[PATCH 00/12] vinyl: allow to modify format of non-empty spaces

[PATCH 00/12] vinyl: allow to modify format of non-empty spaces

[Vladimir Davydov]

This patch set makes it possible to modify the format of a non-empty
Vinyl space. It is an integral part of Vinyl space ALTER (see #1653).

https://github.com/tarantool/tarantool/issues/1653
https://github.com/tarantool/tarantool/commits/vy-allow-to-change-space-format

Vladimir Davydov (12):
  alter: introduce CheckSpaceFormat AlterSpaceOp for validating format
  alter: fold ModifySpaceFormat into ModifySpace
  alter: move dictionary update from ModifySpace::alter_def to alter
  alter: use space_index instead of index_find where appropriate
  alter: allocate triggers before the point of no return
  space: space_vtab::build_secondary_key => build_index
  space: pass new format instead of new space to
    space_vtab::check_format
  alter: introduce preparation phase
  alter: zap space_def_check_compatibility
  vinyl: remove superfluous ddl checks
  vinyl: force index rebuild if indexed field type is narrowed
  vinyl: allow to modify format of non-empty spaces

 src/box/alter.cc             | 280 +++++++++------
 src/box/index.h              |  13 +
 src/box/index_def.c          |  21 --
 src/box/index_def.h          |  16 -
 src/box/key_def.cc           |  19 -
 src/box/key_def.h            |  13 -
 src/box/memtx_bitset.c       |   2 +
 src/box/memtx_engine.c       |  42 +++
 src/box/memtx_engine.h       |   8 +
 src/box/memtx_hash.c         |   2 +
 src/box/memtx_rtree.c        |  15 +
 src/box/memtx_space.c        |  47 +--
 src/box/memtx_tree.c         |   2 +
 src/box/space.h              |  43 ++-
 src/box/space_def.c          |  49 ---
 src/box/space_def.h          |  26 --
 src/box/sysview_engine.c     |  19 +-
 src/box/sysview_index.c      |  11 +
 src/box/tuple_format.c       |   2 -
 src/box/vinyl.c              | 234 +++++++++----
 src/errinj.h                 |   2 +-
 test/box/alter.result        | 817 +------------------------------------------
 test/box/alter.test.lua      | 319 -----------------
 test/box/alter_limits.result |   9 +-
 test/box/errinj.result       |  34 +-
 test/box/errinj.test.lua     |  10 +-
 test/engine/ddl.result       | 787 ++++++++++++++++++++++++++++++++++++++++-
 test/engine/ddl.test.lua     | 311 +++++++++++++++-
 test/vinyl/ddl.result        | 135 +++----
 test/vinyl/ddl.test.lua      |  37 +-
 test/vinyl/errinj.result     |  75 ++++
 test/vinyl/errinj.test.lua   |  35 ++
 test/vinyl/gh.result         |   2 +-
 33 files changed, 1759 insertions(+), 1678 deletions(-)

-- 
2.11.0

[PATCH 01/12] alter: introduce CheckSpaceFormat AlterSpaceOp for validating format

[Vladimir Davydov]

ModifySpaceFormat combines two functions: tuple dictionary update and
format check. We don't need to update tuple dictionary when we modify
an index, but since we don't have a separate operation for validating
tuple format, we issue ModifySpaceFormat. This is confusing. Let's
delegate format checking to a new operation CheckSpaceFormat.
---
 src/box/alter.cc | 50 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index b9d3c2d5..9fea81ad 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -915,7 +915,6 @@ public:
 	ModifySpaceFormat(struct alter_space *alter, struct space_def *new_def)
 		: AlterSpaceOp(alter), new_dict(NULL), old_dict(NULL),
 		  new_def(new_def) {}
-	virtual void alter(struct alter_space *alter);
 	virtual void alter_def(struct alter_space *alter);
 	virtual void commit(struct alter_space *alter, int64_t lsn);
 	virtual ~ModifySpaceFormat();
@@ -937,21 +936,6 @@ ModifySpaceFormat::alter_def(struct alter_space *alter)
 }
 
 void
-ModifySpaceFormat::alter(struct alter_space *alter)
-{
-	struct space *new_space = alter->new_space;
-	struct space *old_space = alter->old_space;
-	struct tuple_format *new_format = new_space->format;
-	struct tuple_format *old_format = old_space->format;
-	if (old_format != NULL) {
-		assert(new_format != NULL);
-		if (! tuple_format1_can_store_format2_tuples(new_format,
-							     old_format))
-		    space_check_format_xc(new_space, old_space);
-	}
-}
-
-void
 ModifySpaceFormat::commit(struct alter_space *alter, int64_t lsn)
 {
 	(void) alter;
@@ -969,6 +953,33 @@ ModifySpaceFormat::~ModifySpaceFormat()
 	}
 }
 
+/**
+ * This operation does not modify the space, it just checks that
+ * tuples stored in it conform to the new format.
+ */
+class CheckSpaceFormat: public AlterSpaceOp
+{
+public:
+	CheckSpaceFormat(struct alter_space *alter)
+		:AlterSpaceOp(alter) {}
+	virtual void alter(struct alter_space *alter);
+};
+
+void
+CheckSpaceFormat::alter(struct alter_space *alter)
+{
+	struct space *new_space = alter->new_space;
+	struct space *old_space = alter->old_space;
+	struct tuple_format *new_format = new_space->format;
+	struct tuple_format *old_format = old_space->format;
+	if (old_format != NULL) {
+		assert(new_format != NULL);
+		if (!tuple_format1_can_store_format2_tuples(new_format,
+							    old_format))
+		    space_check_format_xc(new_space, old_space);
+	}
+}
+
 /** Change non-essential properties of a space. */
 class ModifySpace: public AlterSpaceOp
 {
@@ -1610,6 +1621,7 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event)
 						     old_space->index_count,
 						     def->fields,
 						     def->field_count);
+		(void) new CheckSpaceFormat(alter);
 		(void) new ModifySpaceFormat(alter, def);
 		(void) new ModifySpace(alter, def);
 		def_guard.is_active = false;
@@ -1798,10 +1810,12 @@ on_replace_dd_index(struct trigger * /* trigger */, void *event)
 						old_index->def);
 			index_def_guard.is_active = false;
 		} else {
-			(void) new ModifySpaceFormat(alter, old_space->def);
 			/*
-			 * Operation can be done without index rebuild.
+			 * Operation can be done without index rebuild,
+			 * but we still need to check that tuples stored
+			 * in the space conform to the new format.
 			 */
+			(void) new CheckSpaceFormat(alter);
 			(void) new ModifyIndex(alter, index_def,
 					       old_index->def);
 			index_def_guard.is_active = false;
-- 
2.11.0

Re: [PATCH 01/12] alter: introduce CheckSpaceFormat AlterSpaceOp for validating format

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> ---
>  src/box/alter.cc | 50 ++++++++++++++++++++++++++++++++------------------
>  1 file changed, 32 insertions(+), 18 deletions(-)

OK to push.

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 02/12] alter: fold ModifySpaceFormat into ModifySpace

[Vladimir Davydov]

ModifySpaceFormat and ModifySpace are always called together, whenever
a space definition is updated. Let's merge them. No functional changes.
---
 src/box/alter.cc | 110 ++++++++++++++++++++-----------------------------------
 1 file changed, 40 insertions(+), 70 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 9fea81ad..0af4ac09 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -886,74 +886,6 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 /* {{{ AlterSpaceOp descendants - alter operations, such as Add/Drop index */
 
 /**
- * The operation is executed on each space format change.
- * Now the single purpose is to update an old field names
- * dictionary, used by old space formats, and use it in a new
- * formats (vinyl creates many formats, not one).
- */
-class ModifySpaceFormat: public AlterSpaceOp
-{
-	/**
-	 * Newely created field dictionary. When new space_def is
-	 * created, it allocates new dictionary. Alter moves new
-	 * names into an old dictionary and deletes new one.
-	 */
-	struct tuple_dictionary *new_dict;
-	/**
-	 * Old tuple dictionary stored to rollback in destructor,
-	 * if an exception had been raised after alter_def(), but
-	 * before alter().
-	 */
-	struct tuple_dictionary *old_dict;
-	/**
-	 * New space definition. It can not be got from alter,
-	 * because alter_def() is called before
-	 * ModifySpace::alter_def().
-	 */
-	struct space_def *new_def;
-public:
-	ModifySpaceFormat(struct alter_space *alter, struct space_def *new_def)
-		: AlterSpaceOp(alter), new_dict(NULL), old_dict(NULL),
-		  new_def(new_def) {}
-	virtual void alter_def(struct alter_space *alter);
-	virtual void commit(struct alter_space *alter, int64_t lsn);
-	virtual ~ModifySpaceFormat();
-};
-
-void
-ModifySpaceFormat::alter_def(struct alter_space *alter)
-{
-	/*
-	 * Move new names into an old dictionary, which already is
-	 * referenced by existing tuple formats. New dictionary
-	 * object is deleted later, in destructor.
-	 */
-	new_dict = new_def->dict;
-	old_dict = alter->old_space->def->dict;
-	tuple_dictionary_swap(new_dict, old_dict);
-	new_def->dict = old_dict;
-	tuple_dictionary_ref(old_dict);
-}
-
-void
-ModifySpaceFormat::commit(struct alter_space *alter, int64_t lsn)
-{
-	(void) alter;
-	(void) lsn;
-	old_dict = NULL;
-}
-
-ModifySpaceFormat::~ModifySpaceFormat()
-{
-	if (new_dict != NULL) {
-		/* Return old names into the old dict. */
-		if (old_dict != NULL)
-			tuple_dictionary_swap(new_dict, old_dict);
-		tuple_dictionary_unref(new_dict);
-	}
-}
-
-/**
  * This operation does not modify the space, it just checks that
  * tuples stored in it conform to the new format.
  */
@@ -986,9 +918,22 @@ class ModifySpace: public AlterSpaceOp
 public:
 	ModifySpace(struct alter_space *alter, struct space_def *def_arg)
 		:AlterSpaceOp(alter), def(def_arg) {}
+	/**
+	 * Newely created field dictionary. When new space_def is
+	 * created, it allocates new dictionary. Alter moves new
+	 * names into an old dictionary and deletes new one.
+	 */
+	struct tuple_dictionary *new_dict;
+	/**
+	 * Old tuple dictionary stored to rollback in destructor,
+	 * if an exception had been raised after alter_def(), but
+	 * before alter().
+	 */
+	struct tuple_dictionary *old_dict;
 	/* New space definition. */
 	struct space_def *def;
 	virtual void alter_def(struct alter_space *alter);
+	virtual void commit(struct alter_space *alter, int64_t signature);
 	virtual ~ModifySpace();
 };
 
@@ -996,13 +941,39 @@ public:
 void
 ModifySpace::alter_def(struct alter_space *alter)
 {
+	/*
+	 * Move new names into an old dictionary, which already is
+	 * referenced by existing tuple formats. New dictionary
+	 * object is deleted later, in destructor.
+	 */
+	new_dict = def->dict;
+	old_dict = alter->old_space->def->dict;
+	tuple_dictionary_swap(new_dict, old_dict);
+	def->dict = old_dict;
+	tuple_dictionary_ref(old_dict);
+
 	space_def_delete(alter->space_def);
 	alter->space_def = def;
 	/* Now alter owns the def. */
 	def = NULL;
 }
 
-ModifySpace::~ModifySpace() {
+void
+ModifySpace::commit(struct alter_space *alter, int64_t signature)
+{
+	(void) alter;
+	(void) signature;
+	old_dict = NULL;
+}
+
+ModifySpace::~ModifySpace()
+{
+	if (new_dict != NULL) {
+		/* Return old names into the old dict. */
+		if (old_dict != NULL)
+			tuple_dictionary_swap(new_dict, old_dict);
+		tuple_dictionary_unref(new_dict);
+	}
 	if (def != NULL)
 		space_def_delete(def);
 }
@@ -1622,7 +1593,6 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event)
 						     def->fields,
 						     def->field_count);
 		(void) new CheckSpaceFormat(alter);
-		(void) new ModifySpaceFormat(alter, def);
 		(void) new ModifySpace(alter, def);
 		def_guard.is_active = false;
 		/* Create MoveIndex ops for all space indexes. */
-- 
2.11.0

Re: [PATCH 02/12] alter: fold ModifySpaceFormat into ModifySpace

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> ---
>  src/box/alter.cc | 110 ++++++++++++++++++++-----------------------------------
>  1 file changed, 40 insertions(+), 70 deletions(-)

OK to push.


-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 03/12] alter: move dictionary update from ModifySpace::alter_def to alter

[Vladimir Davydov]

We may yield while rebuilding a vinyl index hence we must not modify
the old space before we are done building all indexes, otherwise the
user might see an inconsistent state while a space is being altered.
Currently, this requirement doesn't hold - we both modify the old space
and build the new indexes in AlterSpaceOp::alter. We need to introduce
AlterSpaceOp::prepare method that would perform all yielding operations
and forbid yielding in AlterSpaceOp::alter.

Dictionary update, which is currently done by ModifySpace::alter_def,
modifies the old space hence it should live in ModifySpace::alter.
Let's move it there.
---
 src/box/alter.cc       | 45 +++++++++++++++++++--------------------------
 src/box/tuple_format.c |  2 --
 2 files changed, 19 insertions(+), 28 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 0af4ac09..ec7af24b 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -923,17 +923,12 @@ public:
 	 * created, it allocates new dictionary. Alter moves new
 	 * names into an old dictionary and deletes new one.
 	 */
-	struct tuple_dictionary *new_dict;
-	/**
-	 * Old tuple dictionary stored to rollback in destructor,
-	 * if an exception had been raised after alter_def(), but
-	 * before alter().
-	 */
-	struct tuple_dictionary *old_dict;
+	struct tuple_dictionary *dict;
 	/* New space definition. */
 	struct space_def *def;
 	virtual void alter_def(struct alter_space *alter);
-	virtual void commit(struct alter_space *alter, int64_t signature);
+	virtual void alter(struct alter_space *alter);
+	virtual void rollback(struct alter_space *alter);
 	virtual ~ModifySpace();
 };
 
@@ -942,15 +937,13 @@ void
 ModifySpace::alter_def(struct alter_space *alter)
 {
 	/*
-	 * Move new names into an old dictionary, which already is
-	 * referenced by existing tuple formats. New dictionary
-	 * object is deleted later, in destructor.
+	 * Use the old dictionary for the new space, because
+	 * it is already referenced by existing tuple formats.
+	 * We will update it in place in ModifySpace::alter.
 	 */
-	new_dict = def->dict;
-	old_dict = alter->old_space->def->dict;
-	tuple_dictionary_swap(new_dict, old_dict);
-	def->dict = old_dict;
-	tuple_dictionary_ref(old_dict);
+	dict = def->dict;
+	def->dict = alter->old_space->def->dict;
+	tuple_dictionary_ref(def->dict);
 
 	space_def_delete(alter->space_def);
 	alter->space_def = def;
@@ -959,21 +952,21 @@ ModifySpace::alter_def(struct alter_space *alter)
 }
 
 void
-ModifySpace::commit(struct alter_space *alter, int64_t signature)
+ModifySpace::alter(struct alter_space *alter)
 {
-	(void) alter;
-	(void) signature;
-	old_dict = NULL;
+	tuple_dictionary_swap(alter->new_space->def->dict, dict);
+}
+
+void
+ModifySpace::rollback(struct alter_space *alter)
+{
+	tuple_dictionary_swap(alter->new_space->def->dict, dict);
 }
 
 ModifySpace::~ModifySpace()
 {
-	if (new_dict != NULL) {
-		/* Return old names into the old dict. */
-		if (old_dict != NULL)
-			tuple_dictionary_swap(new_dict, old_dict);
-		tuple_dictionary_unref(new_dict);
-	}
+	if (dict != NULL)
+		tuple_dictionary_unref(dict);
 	if (def != NULL)
 		space_def_delete(def);
 }
diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c
index d3a7702d..11481473 100644
--- a/src/box/tuple_format.c
+++ b/src/box/tuple_format.c
@@ -262,8 +262,6 @@ tuple_format_new(struct tuple_format_vtab *vtab, struct key_def * const *keys,
 		 const struct field_def *space_fields,
 		 uint32_t space_field_count, struct tuple_dictionary *dict)
 {
-	assert((dict == NULL && space_field_count == 0) ||
-	       (dict != NULL && space_field_count == dict->name_count));
 	struct tuple_format *format =
 		tuple_format_alloc(keys, key_count, space_field_count, dict);
 	if (format == NULL)
-- 
2.11.0

Re: [PATCH 03/12] alter: move dictionary update from ModifySpace::alter_def to alter

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> -	struct tuple_dictionary *new_dict;
> -	/**
> -	 * Old tuple dictionary stored to rollback in destructor,
> -	 * if an exception had been raised after alter_def(), but
> -	 * before alter().
> -	 */
> -	struct tuple_dictionary *old_dict;
> +	struct tuple_dictionary *dict;

Please provide a comment. 

>  	/*
> -	 * Move new names into an old dictionary, which already is
> -	 * referenced by existing tuple formats. New dictionary
> -	 * object is deleted later, in destructor.
> +	 * Use the old dictionary for the new space, because
> +	 * it is already referenced by existing tuple formats.
> +	 * We will update it in place in ModifySpace::alter.
>  	 */

I don't understand this comment. Naming the variable "dict" didn't
help. Not having a comment for the variable where it was declared
didn't help :(

I do understand that your strategy is to swap the new space' dict
with the old space dict and hold the new space dict in a temp
variable until possible rollback. But I don't understand why you
had to change it. The changeset comment says nothing about
changing the way you modify the format, it only mentions changes
in the timing.

> -	new_dict = def->dict;
> -	old_dict = alter->old_space->def->dict;
> -	tuple_dictionary_swap(new_dict, old_dict);
> -	def->dict = old_dict;
> -	tuple_dictionary_ref(old_dict);
> +	dict = def->dict;
> +	def->dict = alter->old_space->def->dict;
> +	tuple_dictionary_ref(def->dict);
>  
>  	space_def_delete(alter->space_def);
>  	alter->space_def = def;
> @@ -959,21 +952,21 @@ ModifySpace::alter_def(struct alter_space *alter)
>  }
>  
>  void
> -ModifySpace::commit(struct alter_space *alter, int64_t signature)
> +ModifySpace::alter(struct alter_space *alter)
>  {
> -	(void) alter;
> -	(void) signature;
> -	old_dict = NULL;
> +	tuple_dictionary_swap(alter->new_space->def->dict, dict);
> +}
> +
> +void
> +ModifySpace::rollback(struct alter_space *alter)
> +{
> +	tuple_dictionary_swap(alter->new_space->def->dict, dict);
>  }
>  
>  ModifySpace::~ModifySpace()
>  {
> -	if (new_dict != NULL) {
> -		/* Return old names into the old dict. */
> -		if (old_dict != NULL)
> -			tuple_dictionary_swap(new_dict, old_dict);
> -		tuple_dictionary_unref(new_dict);
> -	}
> +	if (dict != NULL)
> +		tuple_dictionary_unref(dict);
>  	if (def != NULL)
>  		space_def_delete(def);
>  }
> diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c
> index d3a7702d..11481473 100644
> --- a/src/box/tuple_format.c
> +++ b/src/box/tuple_format.c
> @@ -262,8 +262,6 @@ tuple_format_new(struct tuple_format_vtab *vtab, struct key_def * const *keys,
>  		 const struct field_def *space_fields,
>  		 uint32_t space_field_count, struct tuple_dictionary *dict)
>  {
> -	assert((dict == NULL && space_field_count == 0) ||
> -	       (dict != NULL && space_field_count == dict->name_count));
>  	struct tuple_format *format =
>  		tuple_format_alloc(keys, key_count, space_field_count, dict);
>  	if (format == NULL)
> -- 
> 2.11.0
> 

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

Re: [PATCH 03/12] alter: move dictionary update from ModifySpace::alter_def to alter

[Vladimir Davydov]

On Mon, Apr 09, 2018 at 11:32:40PM +0300, Konstantin Osipov wrote:
> * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > -	struct tuple_dictionary *new_dict;
> > -	/**
> > -	 * Old tuple dictionary stored to rollback in destructor,
> > -	 * if an exception had been raised after alter_def(), but
> > -	 * before alter().
> > -	 */
> > -	struct tuple_dictionary *old_dict;
> > +	struct tuple_dictionary *dict;
> 
> Please provide a comment. 

There is a comment left from 'new_dict'. It just didn't get into the
diff. Here is the omitted context:

class ModifySpace: public AlterSpaceOp
{
public:
        ModifySpace(struct alter_space *alter, struct space_def *def_arg)
                :AlterSpaceOp(alter), def(def_arg) {}
        /**
         * Newely created field dictionary. When new space_def is
         * created, it allocates new dictionary. Alter moves new
         * names into an old dictionary and deletes new one.
         */
        struct tuple_dictionary *dict;
        /* New space definition. */
        struct space_def *def;
        virtual void alter_def(struct alter_space *alter);
        virtual void alter(struct alter_space *alter);
        virtual void rollback(struct alter_space *alter);
        virtual ~ModifySpace();
};

> 
> >  	/*
> > -	 * Move new names into an old dictionary, which already is
> > -	 * referenced by existing tuple formats. New dictionary
> > -	 * object is deleted later, in destructor.
> > +	 * Use the old dictionary for the new space, because
> > +	 * it is already referenced by existing tuple formats.
> > +	 * We will update it in place in ModifySpace::alter.
> >  	 */
> 
> I don't understand this comment. Naming the variable "dict" didn't
> help. Not having a comment for the variable where it was declared
> didn't help :(
> 
> I do understand that your strategy is to swap the new space' dict
> with the old space dict and hold the new space dict in a temp
> variable until possible rollback. But I don't understand why you
> had to change it. The changeset comment says nothing about
> changing the way you modify the format, it only mentions changes
> in the timing.

Because I did only change the timing. The logic behind tuple dictionary
update was originally introduced by commit 9f6113abb ("alter: introduce
ModifySpaceFormat alter operation") and I didn't change it.

We need to swap the content of tuple dictionary, because the old space's
dictionary is referenced by existing tuples and we need to update tuple
field names for them too. We can't just update a pointer in tuple_format
because there may be tuples referencing formats left from prevoius ALTER
operations and we don't keep track of those. So whenever we alter a
space definition, we use the dictionary of the old space when creating a
new space and swap their contents on success to update names for all
tuples stored in the space.

> 
> > -	new_dict = def->dict;
> > -	old_dict = alter->old_space->def->dict;
> > -	tuple_dictionary_swap(new_dict, old_dict);
> > -	def->dict = old_dict;
> > -	tuple_dictionary_ref(old_dict);
> > +	dict = def->dict;
> > +	def->dict = alter->old_space->def->dict;
> > +	tuple_dictionary_ref(def->dict);
> >  
> >  	space_def_delete(alter->space_def);
> >  	alter->space_def = def;
> > @@ -959,21 +952,21 @@ ModifySpace::alter_def(struct alter_space *alter)
> >  }
> >  
> >  void
> > -ModifySpace::commit(struct alter_space *alter, int64_t signature)
> > +ModifySpace::alter(struct alter_space *alter)
> >  {
> > -	(void) alter;
> > -	(void) signature;
> > -	old_dict = NULL;
> > +	tuple_dictionary_swap(alter->new_space->def->dict, dict);
> > +}
> > +
> > +void
> > +ModifySpace::rollback(struct alter_space *alter)
> > +{
> > +	tuple_dictionary_swap(alter->new_space->def->dict, dict);
> >  }
> >  
> >  ModifySpace::~ModifySpace()
> >  {
> > -	if (new_dict != NULL) {
> > -		/* Return old names into the old dict. */
> > -		if (old_dict != NULL)
> > -			tuple_dictionary_swap(new_dict, old_dict);
> > -		tuple_dictionary_unref(new_dict);
> > -	}
> > +	if (dict != NULL)
> > +		tuple_dictionary_unref(dict);
> >  	if (def != NULL)
> >  		space_def_delete(def);
> >  }

Re: [PATCH 03/12] alter: move dictionary update from ModifySpace::alter_def to alter

[Vladimir Davydov]

On Mon, Apr 09, 2018 at 11:32:40PM +0300, Konstantin Osipov wrote:
> * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > -	struct tuple_dictionary *new_dict;
> > -	/**
> > -	 * Old tuple dictionary stored to rollback in destructor,
> > -	 * if an exception had been raised after alter_def(), but
> > -	 * before alter().
> > -	 */
> > -	struct tuple_dictionary *old_dict;
> > +	struct tuple_dictionary *dict;
> 
> Please provide a comment. 
> 
> >  	/*
> > -	 * Move new names into an old dictionary, which already is
> > -	 * referenced by existing tuple formats. New dictionary
> > -	 * object is deleted later, in destructor.
> > +	 * Use the old dictionary for the new space, because
> > +	 * it is already referenced by existing tuple formats.
> > +	 * We will update it in place in ModifySpace::alter.
> >  	 */
> 
> I don't understand this comment. Naming the variable "dict" didn't
> help. Not having a comment for the variable where it was declared
> didn't help :(
> 
> I do understand that your strategy is to swap the new space' dict
> with the old space dict and hold the new space dict in a temp
> variable until possible rollback. But I don't understand why you
> had to change it. The changeset comment says nothing about
> changing the way you modify the format, it only mentions changes
> in the timing.
> 
> > -	new_dict = def->dict;
> > -	old_dict = alter->old_space->def->dict;
> > -	tuple_dictionary_swap(new_dict, old_dict);
> > -	def->dict = old_dict;
> > -	tuple_dictionary_ref(old_dict);
> > +	dict = def->dict;
> > +	def->dict = alter->old_space->def->dict;
> > +	tuple_dictionary_ref(def->dict);
> >  
> >  	space_def_delete(alter->space_def);
> >  	alter->space_def = def;

Fixed as discussed f2f. Please take a look. Note, the patch hasn't been
updated on the branch yet, don't push it.

From 71e9dff0d3666a9ac02739d5754bd8d028134c9f Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Wed, 4 Apr 2018 16:02:07 +0300
Subject: [PATCH] alter: move dictionary update from ModifySpace::alter_def to
 alter

We may yield while rebuilding a vinyl index hence we must not modify
the old space before we are done building all indexes, otherwise the
user might see an inconsistent state while a space is being altered.
Currently, this requirement doesn't hold - we both modify the old space
and build the new indexes in AlterSpaceOp::alter. We need to introduce
AlterSpaceOp::prepare method that would perform all yielding operations
and forbid yielding in AlterSpaceOp::alter.

Dictionary update, which is currently done by ModifySpace::alter_def,
modifies the old space hence it should live in ModifySpace::alter.
Let's move it there.

While we are at it, let's also rename ModifySpace::def to new_def,
because it denotes the new space definition.

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 0af4ac09..6e351cf5 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -916,24 +916,19 @@ CheckSpaceFormat::alter(struct alter_space *alter)
 class ModifySpace: public AlterSpaceOp
 {
 public:
-	ModifySpace(struct alter_space *alter, struct space_def *def_arg)
-		:AlterSpaceOp(alter), def(def_arg) {}
+	ModifySpace(struct alter_space *alter, struct space_def *def)
+		:AlterSpaceOp(alter), new_def(def) {}
 	/**
 	 * Newely created field dictionary. When new space_def is
 	 * created, it allocates new dictionary. Alter moves new
 	 * names into an old dictionary and deletes new one.
 	 */
 	struct tuple_dictionary *new_dict;
-	/**
-	 * Old tuple dictionary stored to rollback in destructor,
-	 * if an exception had been raised after alter_def(), but
-	 * before alter().
-	 */
-	struct tuple_dictionary *old_dict;
 	/* New space definition. */
-	struct space_def *def;
+	struct space_def *new_def;
 	virtual void alter_def(struct alter_space *alter);
-	virtual void commit(struct alter_space *alter, int64_t signature);
+	virtual void alter(struct alter_space *alter);
+	virtual void rollback(struct alter_space *alter);
 	virtual ~ModifySpace();
 };
 
@@ -942,40 +937,43 @@ void
 ModifySpace::alter_def(struct alter_space *alter)
 {
 	/*
-	 * Move new names into an old dictionary, which already is
-	 * referenced by existing tuple formats. New dictionary
-	 * object is deleted later, in destructor.
+	 * Use the old dictionary for the new space, because
+	 * it is already referenced by existing tuple formats.
+	 * We will update it in place in ModifySpace::alter.
 	 */
-	new_dict = def->dict;
-	old_dict = alter->old_space->def->dict;
-	tuple_dictionary_swap(new_dict, old_dict);
-	def->dict = old_dict;
-	tuple_dictionary_ref(old_dict);
+	new_dict = new_def->dict;
+	new_def->dict = alter->old_space->def->dict;
+	tuple_dictionary_ref(new_def->dict);
 
 	space_def_delete(alter->space_def);
-	alter->space_def = def;
+	alter->space_def = new_def;
 	/* Now alter owns the def. */
-	def = NULL;
+	new_def = NULL;
+}
+
+void
+ModifySpace::alter(struct alter_space *alter)
+{
+	/*
+	 * Move new names into an old dictionary, which already is
+	 * referenced by existing tuple formats. New dictionary
+	 * object is deleted later, in destructor.
+	 */
+	tuple_dictionary_swap(alter->new_space->def->dict, new_dict);
 }
 
 void
-ModifySpace::commit(struct alter_space *alter, int64_t signature)
+ModifySpace::rollback(struct alter_space *alter)
 {
-	(void) alter;
-	(void) signature;
-	old_dict = NULL;
+	tuple_dictionary_swap(alter->new_space->def->dict, new_dict);
 }
 
 ModifySpace::~ModifySpace()
 {
-	if (new_dict != NULL) {
-		/* Return old names into the old dict. */
-		if (old_dict != NULL)
-			tuple_dictionary_swap(new_dict, old_dict);
+	if (new_dict != NULL)
 		tuple_dictionary_unref(new_dict);
-	}
-	if (def != NULL)
-		space_def_delete(def);
+	if (new_def != NULL)
+		space_def_delete(new_def);
 }
 
 /** DropIndex - remove an index from space. */
diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c
index d3a7702d..11481473 100644
--- a/src/box/tuple_format.c
+++ b/src/box/tuple_format.c
@@ -262,8 +262,6 @@ tuple_format_new(struct tuple_format_vtab *vtab, struct key_def * const *keys,
 		 const struct field_def *space_fields,
 		 uint32_t space_field_count, struct tuple_dictionary *dict)
 {
-	assert((dict == NULL && space_field_count == 0) ||
-	       (dict != NULL && space_field_count == dict->name_count));
 	struct tuple_format *format =
 		tuple_format_alloc(keys, key_count, space_field_count, dict);
 	if (format == NULL)

[PATCH 04/12] alter: use space_index instead of index_find where appropriate

[Vladimir Davydov]

Using index_find_xc() in AlterSpaceOp::commit and rollback is confusing,
because these functions may not throw. Let's use space_index instead as
we are sure that the index we are looking for must exist. While we are
at it, add some missing assertions.
---
 src/box/alter.cc | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index ec7af24b..36310f1c 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -1006,8 +1006,9 @@ DropIndex::alter(struct alter_space *alter)
 void
 DropIndex::commit(struct alter_space *alter, int64_t /* signature */)
 {
-	struct index *index = index_find_xc(alter->old_space,
-					    old_index_def->iid);
+	struct index *index = space_index(alter->old_space,
+					  old_index_def->iid);
+	assert(index != NULL);
 	index_commit_drop(index);
 }
 
@@ -1105,6 +1106,7 @@ ModifyIndex::commit(struct alter_space *alter, int64_t signature)
 {
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
+	assert(new_index != NULL);
 	index_commit_modify(new_index, signature);
 }
 
@@ -1186,8 +1188,9 @@ CreateIndex::alter(struct alter_space *alter)
 	/**
 	 * Get the new index and build it.
 	 */
-	struct index *new_index = index_find_xc(alter->new_space,
-						new_index_def->iid);
+	struct index *new_index = space_index(alter->new_space,
+					      new_index_def->iid);
+	assert(new_index != NULL);
 	space_build_secondary_key_xc(alter->new_space,
 				     alter->new_space, new_index);
 }
@@ -1195,16 +1198,18 @@ CreateIndex::alter(struct alter_space *alter)
 void
 CreateIndex::commit(struct alter_space *alter, int64_t signature)
 {
-	struct index *new_index = index_find_xc(alter->new_space,
-						new_index_def->iid);
+	struct index *new_index = space_index(alter->new_space,
+					      new_index_def->iid);
+	assert(new_index != NULL);
 	index_commit_create(new_index, signature);
 }
 
 void
 CreateIndex::rollback(struct alter_space *alter)
 {
-	struct index *new_index = index_find_xc(alter->new_space,
-						new_index_def->iid);
+	struct index *new_index = space_index(alter->new_space,
+					      new_index_def->iid);
+	assert(new_index != NULL);
 	index_abort_create(new_index);
 }
 
@@ -1269,8 +1274,10 @@ RebuildIndex::commit(struct alter_space *alter, int64_t signature)
 {
 	struct index *old_index = space_index(alter->old_space,
 					      old_index_def->iid);
+	assert(old_index != NULL);
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
+	assert(new_index != NULL);
 	index_commit_drop(old_index);
 	index_commit_create(new_index, signature);
 }
@@ -1280,6 +1287,7 @@ RebuildIndex::rollback(struct alter_space *alter)
 {
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
+	assert(new_index != NULL);
 	index_abort_create(new_index);
 }
 
-- 
2.11.0

Re: [PATCH 04/12] alter: use space_index instead of index_find where appropriate

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> Using index_find_xc() in AlterSpaceOp::commit and rollback is
> confusing, because these functions may not throw. Let's use
> space_index instead as we are sure that the index we are looking
> for must exist. While we are at it, add some missing assertions.

OK to push.


-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 05/12] alter: allocate triggers before the point of no return

[Vladimir Davydov]

Trigger allocation, as any other memory allocation, is allowed to fail.
If this happens in alter_space_do, the space will be left in an
inconsistent state. Let's move trigger allocation to the beginning of
alter_space_do and add a comment denoting the point of no return.
---
 src/box/alter.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 36310f1c..9d0c4c23 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -803,6 +803,11 @@ alter_space_rollback(struct trigger *trigger, void * /* event */)
 static void
 alter_space_do(struct txn *txn, struct alter_space *alter)
 {
+	/* Prepare triggers while we may fail. */
+	struct trigger *on_commit, *on_rollback;
+	on_commit = txn_alter_trigger_new(alter_space_commit, alter);
+	on_rollback = txn_alter_trigger_new(alter_space_rollback, alter);
+
 	/* Create a definition of the new space. */
 	space_dump_def(alter->old_space, &alter->key_list);
 	class AlterSpaceOp *op;
@@ -853,6 +858,11 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 		throw;
 	}
 
+	/*
+	 * This function must not throw exceptions or yield after
+	 * this point.
+	 */
+
 	/* Rebuild index maps once for all indexes. */
 	space_fill_index_map(alter->old_space);
 	space_fill_index_map(alter->new_space);
@@ -873,11 +883,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 	 * finish or rollback the DDL depending on the results of
 	 * writing to WAL.
 	 */
-	struct trigger *on_commit =
-		txn_alter_trigger_new(alter_space_commit, alter);
 	txn_on_commit(txn, on_commit);
-	struct trigger *on_rollback =
-		txn_alter_trigger_new(alter_space_rollback, alter);
 	txn_on_rollback(txn, on_rollback);
 }
 
-- 
2.11.0

Re: [PATCH 05/12] alter: allocate triggers before the point of no return

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> Trigger allocation, as any other memory allocation, is allowed to fail.
> If this happens in alter_space_do, the space will be left in an
> inconsistent state. Let's move trigger allocation to the beginning of
> alter_space_do and add a comment denoting the point of no return.

Previously we would reference the allocated trigger immediately 
in txn_on_commit() /txn_on_rollback(), the changed code leaks
memory in case of any exception between allocation and "point of no
return".

Please add guards.

> ---
>  src/box/alter.cc | 14 ++++++++++----
>  1 file changed, 10 insertions(+), 4 deletions(-)
> 
> diff --git a/src/box/alter.cc b/src/box/alter.cc
> index 36310f1c..9d0c4c23 100644
> --- a/src/box/alter.cc
> +++ b/src/box/alter.cc
> @@ -803,6 +803,11 @@ alter_space_rollback(struct trigger *trigger, void * /* event */)
>  static void
>  alter_space_do(struct txn *txn, struct alter_space *alter)
>  {
> +	/* Prepare triggers while we may fail. */
> +	struct trigger *on_commit, *on_rollback;
> +	on_commit = txn_alter_trigger_new(alter_space_commit, alter);
> +	on_rollback = txn_alter_trigger_new(alter_space_rollback, alter);
> +
>  	/* Create a definition of the new space. */
>  	space_dump_def(alter->old_space, &alter->key_list);
>  	class AlterSpaceOp *op;
> @@ -853,6 +858,11 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
>  		throw;
>  	}
>  
> +	/*
> +	 * This function must not throw exceptions or yield after
> +	 * this point.
> +	 */
> +
>  	/* Rebuild index maps once for all indexes. */
>  	space_fill_index_map(alter->old_space);
>  	space_fill_index_map(alter->new_space);
> @@ -873,11 +883,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
>  	 * finish or rollback the DDL depending on the results of
>  	 * writing to WAL.
>  	 */
> -	struct trigger *on_commit =
> -		txn_alter_trigger_new(alter_space_commit, alter);
>  	txn_on_commit(txn, on_commit);
> -	struct trigger *on_rollback =
> -		txn_alter_trigger_new(alter_space_rollback, alter);
>  	txn_on_rollback(txn, on_rollback);
>  }
>  
> -- 
> 2.11.0
> 

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

Re: [PATCH 05/12] alter: allocate triggers before the point of no return

[Vladimir Davydov]

On Mon, Apr 09, 2018 at 11:36:02PM +0300, Konstantin Osipov wrote:
> * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > Trigger allocation, as any other memory allocation, is allowed to fail.
> > If this happens in alter_space_do, the space will be left in an
> > inconsistent state. Let's move trigger allocation to the beginning of
> > alter_space_do and add a comment denoting the point of no return.
> 
> Previously we would reference the allocated trigger immediately 
> in txn_on_commit() /txn_on_rollback(), the changed code leaks
> memory in case of any exception between allocation and "point of no
> return".
> 
> Please add guards.

We don't need guards for the triggers, because they are allocated on
region - see txn_alter_trigger_new().

> 
> > ---
> >  src/box/alter.cc | 14 ++++++++++----
> >  1 file changed, 10 insertions(+), 4 deletions(-)
> > 
> > diff --git a/src/box/alter.cc b/src/box/alter.cc
> > index 36310f1c..9d0c4c23 100644
> > --- a/src/box/alter.cc
> > +++ b/src/box/alter.cc
> > @@ -803,6 +803,11 @@ alter_space_rollback(struct trigger *trigger, void * /* event */)
> >  static void
> >  alter_space_do(struct txn *txn, struct alter_space *alter)
> >  {
> > +	/* Prepare triggers while we may fail. */
> > +	struct trigger *on_commit, *on_rollback;
> > +	on_commit = txn_alter_trigger_new(alter_space_commit, alter);
> > +	on_rollback = txn_alter_trigger_new(alter_space_rollback, alter);
> > +
> >  	/* Create a definition of the new space. */
> >  	space_dump_def(alter->old_space, &alter->key_list);
> >  	class AlterSpaceOp *op;
> > @@ -853,6 +858,11 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
> >  		throw;
> >  	}
> >  
> > +	/*
> > +	 * This function must not throw exceptions or yield after
> > +	 * this point.
> > +	 */
> > +
> >  	/* Rebuild index maps once for all indexes. */
> >  	space_fill_index_map(alter->old_space);
> >  	space_fill_index_map(alter->new_space);
> > @@ -873,11 +883,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
> >  	 * finish or rollback the DDL depending on the results of
> >  	 * writing to WAL.
> >  	 */
> > -	struct trigger *on_commit =
> > -		txn_alter_trigger_new(alter_space_commit, alter);
> >  	txn_on_commit(txn, on_commit);
> > -	struct trigger *on_rollback =
> > -		txn_alter_trigger_new(alter_space_rollback, alter);
> >  	txn_on_rollback(txn, on_rollback);
> >  }

Re: [PATCH 05/12] alter: allocate triggers before the point of no return

[Vladimir Davydov]

On Tue, Apr 10, 2018 at 10:57:42AM +0300, Vladimir Davydov wrote:
> On Mon, Apr 09, 2018 at 11:36:02PM +0300, Konstantin Osipov wrote:
> > * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > > Trigger allocation, as any other memory allocation, is allowed to fail.
> > > If this happens in alter_space_do, the space will be left in an
> > > inconsistent state. Let's move trigger allocation to the beginning of
> > > alter_space_do and add a comment denoting the point of no return.
> > 
> > Previously we would reference the allocated trigger immediately 
> > in txn_on_commit() /txn_on_rollback(), the changed code leaks
> > memory in case of any exception between allocation and "point of no
> > return".
> > 
> > Please add guards.
> 
> We don't need guards for the triggers, because they are allocated on
> region - see txn_alter_trigger_new().

Added a comment, as discussed:

(not on the branch, don't push)

From d0467518750a6d7aaaf0eb80842ecd6ccc6b2219 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Wed, 4 Apr 2018 18:48:13 +0300
Subject: [PATCH] alter: allocate triggers before the point of no return

Trigger allocation, as any other memory allocation, is allowed to fail.
If this happens in alter_space_do, the space will be left in an
inconsistent state. Let's move trigger allocation to the beginning of
alter_space_do and add a comment denoting the point of no return.

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 96143b95..c4141145 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -803,6 +803,15 @@ alter_space_rollback(struct trigger *trigger, void * /* event */)
 static void
 alter_space_do(struct txn *txn, struct alter_space *alter)
 {
+	/*
+	 * Prepare triggers while we may fail. Note, we don't have to
+	 * free them in case of failure, because they are allocated on
+	 * the region.
+	 */
+	struct trigger *on_commit, *on_rollback;
+	on_commit = txn_alter_trigger_new(alter_space_commit, alter);
+	on_rollback = txn_alter_trigger_new(alter_space_rollback, alter);
+
 	/* Create a definition of the new space. */
 	space_dump_def(alter->old_space, &alter->key_list);
 	class AlterSpaceOp *op;
@@ -853,6 +862,11 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 		throw;
 	}
 
+	/*
+	 * This function must not throw exceptions or yield after
+	 * this point.
+	 */
+
 	/* Rebuild index maps once for all indexes. */
 	space_fill_index_map(alter->old_space);
 	space_fill_index_map(alter->new_space);
@@ -873,11 +887,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 	 * finish or rollback the DDL depending on the results of
 	 * writing to WAL.
 	 */
-	struct trigger *on_commit =
-		txn_alter_trigger_new(alter_space_commit, alter);
 	txn_on_commit(txn, on_commit);
-	struct trigger *on_rollback =
-		txn_alter_trigger_new(alter_space_rollback, alter);
 	txn_on_rollback(txn, on_rollback);
 }
 

[PATCH 06/12] space: space_vtab::build_secondary_key => build_index

[Vladimir Davydov]

The build_secondary_key method of space vtab is used not only for
building secondary indexes, but also for rebuilding primary indexes.
To avoid confusion, let's rename it to build_index and pass to it
the source space, the new index, and the new tuple format.
---
 src/box/alter.cc         | 10 +++++-----
 src/box/memtx_space.c    | 26 ++++++++++++--------------
 src/box/space.h          | 27 +++++++++++++--------------
 src/box/sysview_engine.c | 13 ++++++-------
 src/box/vinyl.c          | 18 ++++++------------
 src/errinj.h             |  2 +-
 test/box/errinj.result   | 34 +++++++++++++++++-----------------
 test/box/errinj.test.lua | 10 +++++-----
 8 files changed, 65 insertions(+), 75 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 9d0c4c23..947716c8 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -1197,8 +1197,8 @@ CreateIndex::alter(struct alter_space *alter)
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_secondary_key_xc(alter->new_space,
-				     alter->new_space, new_index);
+	space_build_index_xc(alter->new_space, new_index,
+			     alter->new_space->format);
 }
 
 void
@@ -1270,9 +1270,9 @@ RebuildIndex::alter(struct alter_space *alter)
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_secondary_key_xc(new_index_def->iid != 0 ?
-				     alter->new_space : alter->old_space,
-				     alter->new_space, new_index);
+	space_build_index_xc(new_index_def->iid != 0 ?
+			     alter->new_space : alter->old_space,
+			     new_index, alter->new_space->format);
 }
 
 void
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index b3e08e49..ca7b5937 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -754,27 +754,26 @@ memtx_init_system_space(struct space *space)
 }
 
 static int
-memtx_space_build_secondary_key(struct space *old_space,
-				struct space *new_space,
-				struct index *new_index)
+memtx_space_build_index(struct space *space, struct index *index,
+			struct tuple_format *format)
 {
 	/**
 	 * If it's a secondary key, and we're not building them
 	 * yet (i.e. it's snapshot recovery for memtx), do nothing.
 	 */
-	if (new_index->def->iid != 0) {
+	if (index->def->iid != 0) {
 		struct memtx_space *memtx_space;
-		memtx_space = (struct memtx_space *)new_space;
+		memtx_space = (struct memtx_space *)space;
 		if (!(memtx_space->replace == memtx_space_replace_all_keys))
 			return 0;
 	}
-	struct index *pk = index_find(old_space, 0);
+	struct index *pk = index_find(space, 0);
 	if (pk == NULL)
 		return -1;
 
-	struct errinj *inj = errinj(ERRINJ_BUILD_SECONDARY, ERRINJ_INT);
-	if (inj != NULL && inj->iparam == (int)new_index->def->iid) {
-		diag_set(ClientError, ER_INJECTION, "buildSecondaryKey");
+	struct errinj *inj = errinj(ERRINJ_BUILD_INDEX, ERRINJ_INT);
+	if (inj != NULL && inj->iparam == (int)index->def->iid) {
+		diag_set(ClientError, ER_INJECTION, "build index");
 		return -1;
 	}
 
@@ -798,15 +797,14 @@ memtx_space_build_secondary_key(struct space *old_space,
 		 * Check that the tuple is OK according to the
 		 * new format.
 		 */
-		rc = tuple_validate(new_space->format, tuple);
+		rc = tuple_validate(format, tuple);
 		if (rc != 0)
 			break;
 		/*
 		 * @todo: better message if there is a duplicate.
 		 */
 		struct tuple *old_tuple;
-		rc = index_replace(new_index, NULL, tuple,
-				   DUP_INSERT, &old_tuple);
+		rc = index_replace(index, NULL, tuple, DUP_INSERT, &old_tuple);
 		if (rc != 0)
 			break;
 		assert(old_tuple == NULL); /* Guaranteed by DUP_INSERT. */
@@ -815,7 +813,7 @@ memtx_space_build_secondary_key(struct space *old_space,
 		 * All tuples stored in a memtx space must be
 		 * referenced by the primary index.
 		 */
-		if (new_index->def->iid == 0)
+		if (index->def->iid == 0)
 			tuple_ref(tuple);
 	}
 	iterator_delete(it);
@@ -851,7 +849,7 @@ static const struct space_vtab memtx_space_vtab = {
 	/* .add_primary_key = */ memtx_space_add_primary_key,
 	/* .drop_primary_key = */ memtx_space_drop_primary_key,
 	/* .check_format  = */ memtx_space_check_format,
-	/* .build_secondary_key = */ memtx_space_build_secondary_key,
+	/* .build_index = */ memtx_space_build_index,
 	/* .swap_index = */ generic_space_swap_index,
 	/* .prepare_alter = */ memtx_space_prepare_alter,
 };
diff --git a/src/box/space.h b/src/box/space.h
index 758d575e..0cc1b00f 100644
--- a/src/box/space.h
+++ b/src/box/space.h
@@ -48,6 +48,7 @@ struct txn;
 struct request;
 struct port;
 struct tuple;
+struct tuple_format;
 
 struct space_vtab {
 	/** Free a space instance. */
@@ -96,13 +97,13 @@ struct space_vtab {
 	int (*check_format)(struct space *new_space,
 			    struct space *old_space);
 	/**
-	 * Called with the new empty secondary index.
-	 * Fill the new index with data from the primary
-	 * key of the space.
+	 * Build a new index, primary or secondary, and fill it
+	 * with tuples stored in the given space. The function is
+	 * supposed to assure that all tuples conform to the new
+	 * format.
 	 */
-	int (*build_secondary_key)(struct space *old_space,
-				   struct space *new_space,
-				   struct index *new_index);
+	int (*build_index)(struct space *space, struct index *index,
+			   struct tuple_format *format);
 	/**
 	 * Exchange two index objects in two spaces. Used
 	 * to update a space with a newly built index, while
@@ -329,12 +330,10 @@ space_drop_primary_key(struct space *space)
 }
 
 static inline int
-space_build_secondary_key(struct space *old_space,
-			  struct space *new_space, struct index *new_index)
+space_build_index(struct space *space, struct index *index,
+		  struct tuple_format *format)
 {
-	assert(old_space->vtab == new_space->vtab);
-	return new_space->vtab->build_secondary_key(old_space,
-						    new_space, new_index);
+	return space->vtab->build_index(space, index, format);
 }
 
 static inline void
@@ -480,10 +479,10 @@ space_check_format_xc(struct space *new_space, struct space *old_space)
 }
 
 static inline void
-space_build_secondary_key_xc(struct space *old_space,
-			     struct space *new_space, struct index *new_index)
+space_build_index_xc(struct space *space, struct index *index,
+		     struct tuple_format *format)
 {
-	if (space_build_secondary_key(old_space, new_space, new_index) != 0)
+	if (space_build_index(space, index, format) != 0)
 		diag_raise();
 }
 
diff --git a/src/box/sysview_engine.c b/src/box/sysview_engine.c
index d28430aa..8cfdeeba 100644
--- a/src/box/sysview_engine.c
+++ b/src/box/sysview_engine.c
@@ -136,13 +136,12 @@ sysview_space_drop_primary_key(struct space *space)
 }
 
 static int
-sysview_space_build_secondary_key(struct space *old_space,
-				  struct space *new_space,
-				  struct index *new_index)
+sysview_space_build_index(struct space *space, struct index *index,
+			  struct tuple_format *format)
 {
-	(void)old_space;
-	(void)new_space;
-	(void)new_index;
+	(void)space;
+	(void)index;
+	(void)format;
 	return 0;
 }
 
@@ -177,7 +176,7 @@ static const struct space_vtab sysview_space_vtab = {
 	/* .add_primary_key = */ sysview_space_add_primary_key,
 	/* .drop_primary_key = */ sysview_space_drop_primary_key,
 	/* .check_format = */ sysview_space_check_format,
-	/* .build_secondary_key = */ sysview_space_build_secondary_key,
+	/* .build_index = */ sysview_space_build_index,
 	/* .swap_index = */ generic_space_swap_index,
 	/* .prepare_alter = */ sysview_space_prepare_alter,
 };
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index e809f6a8..7db7a60f 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1094,11 +1094,10 @@ vinyl_space_drop_primary_key(struct space *space)
 }
 
 static int
-vinyl_space_build_secondary_key(struct space *old_space,
-				struct space *new_space,
-				struct index *new_index)
+vinyl_space_build_index(struct space *space, struct index *index,
+			struct tuple_format *format)
 {
-	(void)old_space;
+	(void)format;
 	/*
 	 * Unlike Memtx, Vinyl does not need building of a secondary index.
 	 * This is true because of two things:
@@ -1113,17 +1112,12 @@ vinyl_space_build_secondary_key(struct space *old_space,
 	 * by itself during WAL recovery.
 	 * III. Vinyl is online. The space is definitely empty and there's
 	 * nothing to build.
-	 *
-	 * When we start to implement alter of non-empty vinyl spaces, it
-	 *  seems that we should call here:
-	 *   Engine::buildSecondaryKey(old_space, new_space, new_index_arg);
-	 *  but aware of three cases mentioned above.
 	 */
-	if (vinyl_index_open(new_index) != 0)
+	if (vinyl_index_open(index) != 0)
 		return -1;
 
 	/* Set pointer to the primary key for the new index. */
-	vy_lsm_update_pk(vy_lsm(new_index), vy_lsm(space_index(new_space, 0)));
+	vy_lsm_update_pk(vy_lsm(index), vy_lsm(space_index(space, 0)));
 	return 0;
 }
 
@@ -3931,7 +3925,7 @@ static const struct space_vtab vinyl_space_vtab = {
 	/* .add_primary_key = */ vinyl_space_add_primary_key,
 	/* .drop_primary_key = */ vinyl_space_drop_primary_key,
 	/* .check_format = */ vinyl_space_check_format,
-	/* .build_secondary_key = */ vinyl_space_build_secondary_key,
+	/* .build_index = */ vinyl_space_build_index,
 	/* .swap_index = */ vinyl_space_swap_index,
 	/* .prepare_alter = */ vinyl_space_prepare_alter,
 };
diff --git a/src/errinj.h b/src/errinj.h
index ab5988cd..2f9d2408 100644
--- a/src/errinj.h
+++ b/src/errinj.h
@@ -102,7 +102,7 @@ struct errinj {
 	_(ERRINJ_XLOG_READ, ERRINJ_INT, {.iparam = -1}) \
 	_(ERRINJ_VYRUN_INDEX_GARBAGE, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_VYRUN_DATA_READ, ERRINJ_BOOL, {.bparam = false}) \
-	_(ERRINJ_BUILD_SECONDARY, ERRINJ_INT, {.iparam = -1}) \
+	_(ERRINJ_BUILD_INDEX, ERRINJ_INT, {.iparam = -1}) \
 	_(ERRINJ_VY_POINT_ITER_WAIT, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_RELAY_EXIT_DELAY, ERRINJ_DOUBLE, {.dparam = 0}) \
 	_(ERRINJ_VY_DELAY_PK_LOOKUP, ERRINJ_BOOL, {.bparam = false}) \
diff --git a/test/box/errinj.result b/test/box/errinj.result
index 92ae7235..958f82dd 100644
--- a/test/box/errinj.result
+++ b/test/box/errinj.result
@@ -56,6 +56,8 @@ errinj.info()
     state: false
   ERRINJ_VY_RUN_WRITE:
     state: false
+  ERRINJ_BUILD_INDEX:
+    state: -1
   ERRINJ_RELAY_FINAL_SLEEP:
     state: false
   ERRINJ_VY_RUN_DISCARD:
@@ -64,30 +66,28 @@ errinj.info()
     state: false
   ERRINJ_LOG_ROTATE:
     state: false
-  ERRINJ_VY_POINT_ITER_WAIT:
-    state: false
   ERRINJ_RELAY_EXIT_DELAY:
     state: 0
   ERRINJ_IPROTO_TX_DELAY:
     state: false
-  ERRINJ_TUPLE_FIELD:
+  ERRINJ_VY_POINT_ITER_WAIT:
     state: false
-  ERRINJ_INDEX_ALLOC:
+  ERRINJ_TUPLE_FIELD:
     state: false
   ERRINJ_XLOG_GARBAGE:
     state: false
-  ERRINJ_VY_RUN_WRITE_TIMEOUT:
-    state: 0
+  ERRINJ_INDEX_ALLOC:
+    state: false
   ERRINJ_RELAY_TIMEOUT:
     state: 0
   ERRINJ_TESTING:
     state: false
-  ERRINJ_VY_LOG_FLUSH:
-    state: false
+  ERRINJ_VY_RUN_WRITE_TIMEOUT:
+    state: 0
   ERRINJ_VY_SQUASH_TIMEOUT:
     state: 0
-  ERRINJ_BUILD_SECONDARY:
-    state: -1
+  ERRINJ_VY_LOG_FLUSH:
+    state: false
   ERRINJ_VY_INDEX_DUMP:
     state: -1
 ...
@@ -949,13 +949,13 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i2.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i2.id)
 ---
 - ok
 ...
 i1:alter{parts = {3, "unsigned"}}
 ---
-- error: Error injection 'buildSecondaryKey'
+- error: Error injection 'build index'
 ...
 _ = collectgarbage('collect')
 ---
@@ -981,13 +981,13 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i3.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i3.id)
 ---
 - ok
 ...
 i1:alter{parts = {4, "unsigned"}}
 ---
-- error: Error injection 'buildSecondaryKey'
+- error: Error injection 'build index'
 ...
 _ = collectgarbage('collect')
 ---
@@ -1013,7 +1013,7 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 ---
 - ok
 ...
@@ -1037,14 +1037,14 @@ s:replace{1, 1}
 ---
 - [1, 1]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', sk.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', sk.id)
 ---
 - ok
 ...
 sk:alter({parts = {2, 'number'}})
 ---
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 ---
 - ok
 ...
diff --git a/test/box/errinj.test.lua b/test/box/errinj.test.lua
index 7dea2769..a3dde845 100644
--- a/test/box/errinj.test.lua
+++ b/test/box/errinj.test.lua
@@ -311,7 +311,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i2.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i2.id)
 
 i1:alter{parts = {3, "unsigned"}}
 
@@ -320,7 +320,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i3.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i3.id)
 
 i1:alter{parts = {4, "unsigned"}}
 
@@ -329,7 +329,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 
 s:drop()
 
@@ -341,9 +341,9 @@ s = box.schema.space.create('test')
 pk = s:create_index('pk')
 sk = s:create_index('sk', {parts = {2, 'unsigned'}})
 s:replace{1, 1}
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', sk.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', sk.id)
 sk:alter({parts = {2, 'number'}})
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 s:drop()
 
 --
-- 
2.11.0

Re: [PATCH 06/12] space: space_vtab::build_secondary_key => build_index

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> -	space_build_secondary_key_xc(new_index_def->iid != 0 ?
> -				     alter->new_space : alter->old_space,
> -				     alter->new_space, new_index);
> +	space_build_index_xc(new_index_def->iid != 0 ?
> +			     alter->new_space : alter->old_space,
> +			     new_index, alter->new_space->format);
>  }

This is confusing, why do you ever need to pass the old space? 
A new index should always be built in the new space, no?

>  	/**
> -	 * Called with the new empty secondary index.
> -	 * Fill the new index with data from the primary
> -	 * key of the space.
> +	 * Build a new index, primary or secondary, and fill it
> +	 * with tuples stored in the given space. The function is
> +	 * supposed to assure that all tuples conform to the new
> +	 * format.
>  	 */
> -	int (*build_secondary_key)(struct space *old_space,
> -				   struct space *new_space,
> -				   struct index *new_index);
> +	int (*build_index)(struct space *space, struct index *index,
> +			   struct tuple_format *format);

Not having parameter 'space' documented, and not having the
relationship between 'space' and 'format' documented (why can't we 
use space->format) doesn't provide more clarity either.

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

Re: [PATCH 06/12] space: space_vtab::build_secondary_key => build_index

[Vladimir Davydov]

On Mon, Apr 09, 2018 at 11:39:25PM +0300, Konstantin Osipov wrote:
> * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > -	space_build_secondary_key_xc(new_index_def->iid != 0 ?
> > -				     alter->new_space : alter->old_space,
> > -				     alter->new_space, new_index);
> > +	space_build_index_xc(new_index_def->iid != 0 ?
> > +			     alter->new_space : alter->old_space,
> > +			     new_index, alter->new_space->format);
> >  }
> 
> This is confusing, why do you ever need to pass the old space? 
> A new index should always be built in the new space, no?

We always build an index *for* the new space, but the 'space' argument
here denotes the space to use as the source for building the new index.
If we are rebuilding the primary key, we have to pass the old space,
because the new space is empty.

> 
> >  	/**
> > -	 * Called with the new empty secondary index.
> > -	 * Fill the new index with data from the primary
> > -	 * key of the space.
> > +	 * Build a new index, primary or secondary, and fill it
> > +	 * with tuples stored in the given space. The function is
> > +	 * supposed to assure that all tuples conform to the new
> > +	 * format.
> >  	 */
> > -	int (*build_secondary_key)(struct space *old_space,
> > -				   struct space *new_space,
> > -				   struct index *new_index);
> > +	int (*build_index)(struct space *space, struct index *index,
> > +			   struct tuple_format *format);
> 
> Not having parameter 'space' documented, and not having the
> relationship between 'space' and 'format' documented (why can't we 
> use space->format) doesn't provide more clarity either.

OK, I will update the comment.

Re: [PATCH 06/12] space: space_vtab::build_secondary_key => build_index

[Vladimir Davydov]

On Tue, Apr 10, 2018 at 11:05:07AM +0300, Vladimir Davydov wrote:
> On Mon, Apr 09, 2018 at 11:39:25PM +0300, Konstantin Osipov wrote:
> > * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > > -	space_build_secondary_key_xc(new_index_def->iid != 0 ?
> > > -				     alter->new_space : alter->old_space,
> > > -				     alter->new_space, new_index);
> > > +	space_build_index_xc(new_index_def->iid != 0 ?
> > > +			     alter->new_space : alter->old_space,
> > > +			     new_index, alter->new_space->format);
> > >  }
> > 
> > This is confusing, why do you ever need to pass the old space? 
> > A new index should always be built in the new space, no?
> 
> We always build an index *for* the new space, but the 'space' argument
> here denotes the space to use as the source for building the new index.
> If we are rebuilding the primary key, we have to pass the old space,
> because the new space is empty.
> 
> > 
> > >  	/**
> > > -	 * Called with the new empty secondary index.
> > > -	 * Fill the new index with data from the primary
> > > -	 * key of the space.
> > > +	 * Build a new index, primary or secondary, and fill it
> > > +	 * with tuples stored in the given space. The function is
> > > +	 * supposed to assure that all tuples conform to the new
> > > +	 * format.
> > >  	 */
> > > -	int (*build_secondary_key)(struct space *old_space,
> > > -				   struct space *new_space,
> > > -				   struct index *new_index);
> > > +	int (*build_index)(struct space *space, struct index *index,
> > > +			   struct tuple_format *format);
> > 
> > Not having parameter 'space' documented, and not having the
> > relationship between 'space' and 'format' documented (why can't we 
> > use space->format) doesn't provide more clarity either.
> 
> OK, I will update the comment.

Done. Please take a look.

(not on the branch, don't push)

From 6d89d2225c116eb86a5cd1735cb4e4537360edf1 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov.dev@gmail.com>
Date: Thu, 5 Apr 2018 13:59:16 +0300
Subject: [PATCH] space: space_vtab::build_secondary_key => build_index

The build_secondary_key method of space vtab is used not only for
building secondary indexes, but also for rebuilding primary indexes.
To avoid confusion, let's rename it to build_index and pass to it
the source space, the new index, and the new tuple format.

diff --git a/src/box/alter.cc b/src/box/alter.cc
index c4141145..11d367a3 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -1206,8 +1206,8 @@ CreateIndex::alter(struct alter_space *alter)
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_secondary_key_xc(alter->new_space,
-				     alter->new_space, new_index);
+	space_build_index_xc(alter->new_space, new_index,
+			     alter->new_space->format);
 }
 
 void
@@ -1279,9 +1279,9 @@ RebuildIndex::alter(struct alter_space *alter)
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_secondary_key_xc(new_index_def->iid != 0 ?
-				     alter->new_space : alter->old_space,
-				     alter->new_space, new_index);
+	space_build_index_xc(new_index_def->iid != 0 ?
+			     alter->new_space : alter->old_space,
+			     new_index, alter->new_space->format);
 }
 
 void
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index b3e08e49..e68f7098 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -754,9 +754,8 @@ memtx_init_system_space(struct space *space)
 }
 
 static int
-memtx_space_build_secondary_key(struct space *old_space,
-				struct space *new_space,
-				struct index *new_index)
+memtx_space_build_index(struct space *src_space, struct index *new_index,
+			struct tuple_format *new_format)
 {
 	/**
 	 * If it's a secondary key, and we're not building them
@@ -764,17 +763,17 @@ memtx_space_build_secondary_key(struct space *old_space,
 	 */
 	if (new_index->def->iid != 0) {
 		struct memtx_space *memtx_space;
-		memtx_space = (struct memtx_space *)new_space;
+		memtx_space = (struct memtx_space *)src_space;
 		if (!(memtx_space->replace == memtx_space_replace_all_keys))
 			return 0;
 	}
-	struct index *pk = index_find(old_space, 0);
+	struct index *pk = index_find(src_space, 0);
 	if (pk == NULL)
 		return -1;
 
-	struct errinj *inj = errinj(ERRINJ_BUILD_SECONDARY, ERRINJ_INT);
+	struct errinj *inj = errinj(ERRINJ_BUILD_INDEX, ERRINJ_INT);
 	if (inj != NULL && inj->iparam == (int)new_index->def->iid) {
-		diag_set(ClientError, ER_INJECTION, "buildSecondaryKey");
+		diag_set(ClientError, ER_INJECTION, "build index");
 		return -1;
 	}
 
@@ -798,7 +797,7 @@ memtx_space_build_secondary_key(struct space *old_space,
 		 * Check that the tuple is OK according to the
 		 * new format.
 		 */
-		rc = tuple_validate(new_space->format, tuple);
+		rc = tuple_validate(new_format, tuple);
 		if (rc != 0)
 			break;
 		/*
@@ -851,7 +850,7 @@ static const struct space_vtab memtx_space_vtab = {
 	/* .add_primary_key = */ memtx_space_add_primary_key,
 	/* .drop_primary_key = */ memtx_space_drop_primary_key,
 	/* .check_format  = */ memtx_space_check_format,
-	/* .build_secondary_key = */ memtx_space_build_secondary_key,
+	/* .build_index = */ memtx_space_build_index,
 	/* .swap_index = */ generic_space_swap_index,
 	/* .prepare_alter = */ memtx_space_prepare_alter,
 };
diff --git a/src/box/space.h b/src/box/space.h
index 758d575e..804c7fb7 100644
--- a/src/box/space.h
+++ b/src/box/space.h
@@ -48,6 +48,7 @@ struct txn;
 struct request;
 struct port;
 struct tuple;
+struct tuple_format;
 
 struct space_vtab {
 	/** Free a space instance. */
@@ -96,13 +97,19 @@ struct space_vtab {
 	int (*check_format)(struct space *new_space,
 			    struct space *old_space);
 	/**
-	 * Called with the new empty secondary index.
-	 * Fill the new index with data from the primary
-	 * key of the space.
+	 * Build a new index, primary or secondary, and fill it
+	 * with tuples stored in the given space. The function is
+	 * supposed to assure that all tuples conform to the new
+	 * format.
+	 *
+	 * @param src_space   space to use as build source
+	 * @param new_index   index to build
+	 * @param new_format  format for validating tuples
+	 * @retval  0         success
+	 * @retval -1         build failed
 	 */
-	int (*build_secondary_key)(struct space *old_space,
-				   struct space *new_space,
-				   struct index *new_index);
+	int (*build_index)(struct space *src_space, struct index *new_index,
+			   struct tuple_format *new_format);
 	/**
 	 * Exchange two index objects in two spaces. Used
 	 * to update a space with a newly built index, while
@@ -329,12 +336,10 @@ space_drop_primary_key(struct space *space)
 }
 
 static inline int
-space_build_secondary_key(struct space *old_space,
-			  struct space *new_space, struct index *new_index)
+space_build_index(struct space *src_space, struct index *new_index,
+		  struct tuple_format *new_format)
 {
-	assert(old_space->vtab == new_space->vtab);
-	return new_space->vtab->build_secondary_key(old_space,
-						    new_space, new_index);
+	return src_space->vtab->build_index(src_space, new_index, new_format);
 }
 
 static inline void
@@ -480,10 +485,10 @@ space_check_format_xc(struct space *new_space, struct space *old_space)
 }
 
 static inline void
-space_build_secondary_key_xc(struct space *old_space,
-			     struct space *new_space, struct index *new_index)
+space_build_index_xc(struct space *src_space, struct index *new_index,
+		     struct tuple_format *new_format)
 {
-	if (space_build_secondary_key(old_space, new_space, new_index) != 0)
+	if (space_build_index(src_space, new_index, new_format) != 0)
 		diag_raise();
 }
 
diff --git a/src/box/sysview_engine.c b/src/box/sysview_engine.c
index d28430aa..47409d23 100644
--- a/src/box/sysview_engine.c
+++ b/src/box/sysview_engine.c
@@ -136,13 +136,12 @@ sysview_space_drop_primary_key(struct space *space)
 }
 
 static int
-sysview_space_build_secondary_key(struct space *old_space,
-				  struct space *new_space,
-				  struct index *new_index)
+sysview_space_build_index(struct space *src_space, struct index *new_index,
+			  struct tuple_format *new_format)
 {
-	(void)old_space;
-	(void)new_space;
+	(void)src_space;
 	(void)new_index;
+	(void)new_format;
 	return 0;
 }
 
@@ -177,7 +176,7 @@ static const struct space_vtab sysview_space_vtab = {
 	/* .add_primary_key = */ sysview_space_add_primary_key,
 	/* .drop_primary_key = */ sysview_space_drop_primary_key,
 	/* .check_format = */ sysview_space_check_format,
-	/* .build_secondary_key = */ sysview_space_build_secondary_key,
+	/* .build_index = */ sysview_space_build_index,
 	/* .swap_index = */ generic_space_swap_index,
 	/* .prepare_alter = */ sysview_space_prepare_alter,
 };
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index e809f6a8..abc30fe8 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1094,11 +1094,10 @@ vinyl_space_drop_primary_key(struct space *space)
 }
 
 static int
-vinyl_space_build_secondary_key(struct space *old_space,
-				struct space *new_space,
-				struct index *new_index)
+vinyl_space_build_index(struct space *src_space, struct index *new_index,
+			struct tuple_format *new_format)
 {
-	(void)old_space;
+	(void)new_format;
 	/*
 	 * Unlike Memtx, Vinyl does not need building of a secondary index.
 	 * This is true because of two things:
@@ -1113,17 +1112,12 @@ vinyl_space_build_secondary_key(struct space *old_space,
 	 * by itself during WAL recovery.
 	 * III. Vinyl is online. The space is definitely empty and there's
 	 * nothing to build.
-	 *
-	 * When we start to implement alter of non-empty vinyl spaces, it
-	 *  seems that we should call here:
-	 *   Engine::buildSecondaryKey(old_space, new_space, new_index_arg);
-	 *  but aware of three cases mentioned above.
 	 */
 	if (vinyl_index_open(new_index) != 0)
 		return -1;
 
 	/* Set pointer to the primary key for the new index. */
-	vy_lsm_update_pk(vy_lsm(new_index), vy_lsm(space_index(new_space, 0)));
+	vy_lsm_update_pk(vy_lsm(new_index), vy_lsm(space_index(src_space, 0)));
 	return 0;
 }
 
@@ -3931,7 +3925,7 @@ static const struct space_vtab vinyl_space_vtab = {
 	/* .add_primary_key = */ vinyl_space_add_primary_key,
 	/* .drop_primary_key = */ vinyl_space_drop_primary_key,
 	/* .check_format = */ vinyl_space_check_format,
-	/* .build_secondary_key = */ vinyl_space_build_secondary_key,
+	/* .build_index = */ vinyl_space_build_index,
 	/* .swap_index = */ vinyl_space_swap_index,
 	/* .prepare_alter = */ vinyl_space_prepare_alter,
 };
diff --git a/src/errinj.h b/src/errinj.h
index ab5988cd..2f9d2408 100644
--- a/src/errinj.h
+++ b/src/errinj.h
@@ -102,7 +102,7 @@ struct errinj {
 	_(ERRINJ_XLOG_READ, ERRINJ_INT, {.iparam = -1}) \
 	_(ERRINJ_VYRUN_INDEX_GARBAGE, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_VYRUN_DATA_READ, ERRINJ_BOOL, {.bparam = false}) \
-	_(ERRINJ_BUILD_SECONDARY, ERRINJ_INT, {.iparam = -1}) \
+	_(ERRINJ_BUILD_INDEX, ERRINJ_INT, {.iparam = -1}) \
 	_(ERRINJ_VY_POINT_ITER_WAIT, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_RELAY_EXIT_DELAY, ERRINJ_DOUBLE, {.dparam = 0}) \
 	_(ERRINJ_VY_DELAY_PK_LOOKUP, ERRINJ_BOOL, {.bparam = false}) \
diff --git a/test/box/errinj.result b/test/box/errinj.result
index 92ae7235..958f82dd 100644
--- a/test/box/errinj.result
+++ b/test/box/errinj.result
@@ -56,6 +56,8 @@ errinj.info()
     state: false
   ERRINJ_VY_RUN_WRITE:
     state: false
+  ERRINJ_BUILD_INDEX:
+    state: -1
   ERRINJ_RELAY_FINAL_SLEEP:
     state: false
   ERRINJ_VY_RUN_DISCARD:
@@ -64,30 +66,28 @@ errinj.info()
     state: false
   ERRINJ_LOG_ROTATE:
     state: false
-  ERRINJ_VY_POINT_ITER_WAIT:
-    state: false
   ERRINJ_RELAY_EXIT_DELAY:
     state: 0
   ERRINJ_IPROTO_TX_DELAY:
     state: false
-  ERRINJ_TUPLE_FIELD:
+  ERRINJ_VY_POINT_ITER_WAIT:
     state: false
-  ERRINJ_INDEX_ALLOC:
+  ERRINJ_TUPLE_FIELD:
     state: false
   ERRINJ_XLOG_GARBAGE:
     state: false
-  ERRINJ_VY_RUN_WRITE_TIMEOUT:
-    state: 0
+  ERRINJ_INDEX_ALLOC:
+    state: false
   ERRINJ_RELAY_TIMEOUT:
     state: 0
   ERRINJ_TESTING:
     state: false
-  ERRINJ_VY_LOG_FLUSH:
-    state: false
+  ERRINJ_VY_RUN_WRITE_TIMEOUT:
+    state: 0
   ERRINJ_VY_SQUASH_TIMEOUT:
     state: 0
-  ERRINJ_BUILD_SECONDARY:
-    state: -1
+  ERRINJ_VY_LOG_FLUSH:
+    state: false
   ERRINJ_VY_INDEX_DUMP:
     state: -1
 ...
@@ -949,13 +949,13 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i2.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i2.id)
 ---
 - ok
 ...
 i1:alter{parts = {3, "unsigned"}}
 ---
-- error: Error injection 'buildSecondaryKey'
+- error: Error injection 'build index'
 ...
 _ = collectgarbage('collect')
 ---
@@ -981,13 +981,13 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i3.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i3.id)
 ---
 - ok
 ...
 i1:alter{parts = {4, "unsigned"}}
 ---
-- error: Error injection 'buildSecondaryKey'
+- error: Error injection 'build index'
 ...
 _ = collectgarbage('collect')
 ---
@@ -1013,7 +1013,7 @@ i3:select{}
   - [2, 3, 1, 2, 10, 10]
   - [1, 4, 3, 4, 10, 10]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 ---
 - ok
 ...
@@ -1037,14 +1037,14 @@ s:replace{1, 1}
 ---
 - [1, 1]
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', sk.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', sk.id)
 ---
 - ok
 ...
 sk:alter({parts = {2, 'number'}})
 ---
 ...
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 ---
 - ok
 ...
diff --git a/test/box/errinj.test.lua b/test/box/errinj.test.lua
index 7dea2769..a3dde845 100644
--- a/test/box/errinj.test.lua
+++ b/test/box/errinj.test.lua
@@ -311,7 +311,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i2.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i2.id)
 
 i1:alter{parts = {3, "unsigned"}}
 
@@ -320,7 +320,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', i3.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', i3.id)
 
 i1:alter{parts = {4, "unsigned"}}
 
@@ -329,7 +329,7 @@ i1:select{}
 i2:select{}
 i3:select{}
 
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 
 s:drop()
 
@@ -341,9 +341,9 @@ s = box.schema.space.create('test')
 pk = s:create_index('pk')
 sk = s:create_index('sk', {parts = {2, 'unsigned'}})
 s:replace{1, 1}
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', sk.id)
+box.error.injection.set('ERRINJ_BUILD_INDEX', sk.id)
 sk:alter({parts = {2, 'number'}})
-box.error.injection.set('ERRINJ_BUILD_SECONDARY', -1)
+box.error.injection.set('ERRINJ_BUILD_INDEX', -1)
 s:drop()
 
 --

[PATCH 07/12] space: pass new format instead of new space to space_vtab::check_format

[Vladimir Davydov]

The check_format method is supposed to check that all tuples stored in
an altered space conform to the new format. Let's change its signature
accordingly.
---
 src/box/alter.cc         |  2 +-
 src/box/memtx_space.c    |  8 ++++----
 src/box/space.h          | 16 +++++++---------
 src/box/sysview_engine.c |  6 +++---
 src/box/vinyl.c          | 10 +++++-----
 5 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 947716c8..9dd8d8d5 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -914,7 +914,7 @@ CheckSpaceFormat::alter(struct alter_space *alter)
 		assert(new_format != NULL);
 		if (!tuple_format1_can_store_format2_tuples(new_format,
 							    old_format))
-		    space_check_format_xc(new_space, old_space);
+		    space_check_format_xc(old_space, new_format);
 	}
 }
 
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index ca7b5937..be687288 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -704,11 +704,11 @@ memtx_space_add_primary_key(struct space *space)
 }
 
 static int
-memtx_space_check_format(struct space *new_space, struct space *old_space)
+memtx_space_check_format(struct space *space, struct tuple_format *format)
 {
-	if (old_space->index_count == 0)
+	if (space->index_count == 0)
 		return 0;
-	struct index *pk = old_space->index[0];
+	struct index *pk = space->index[0];
 	if (index_size(pk) == 0)
 		return 0;
 
@@ -723,7 +723,7 @@ memtx_space_check_format(struct space *new_space, struct space *old_space)
 		 * Check that the tuple is OK according to the
 		 * new format.
 		 */
-		rc = tuple_validate(new_space->format, tuple);
+		rc = tuple_validate(format, tuple);
 		if (rc != 0)
 			break;
 	}
diff --git a/src/box/space.h b/src/box/space.h
index 0cc1b00f..70da58b1 100644
--- a/src/box/space.h
+++ b/src/box/space.h
@@ -91,11 +91,10 @@ struct space_vtab {
 	 */
 	void (*drop_primary_key)(struct space *);
 	/**
-	 * Check that new fields of a space format are
-	 * compatible with existing tuples.
+	 * Check that all tuples stored in a space are compatible
+	 * with the new format.
 	 */
-	int (*check_format)(struct space *new_space,
-			    struct space *old_space);
+	int (*check_format)(struct space *space, struct tuple_format *format);
 	/**
 	 * Build a new index, primary or secondary, and fill it
 	 * with tuples stored in the given space. The function is
@@ -317,10 +316,9 @@ space_add_primary_key(struct space *space)
 }
 
 static inline int
-space_check_format(struct space *new_space, struct space *old_space)
+space_check_format(struct space *space, struct tuple_format *format)
 {
-	assert(old_space->vtab == new_space->vtab);
-	return new_space->vtab->check_format(new_space, old_space);
+	return space->vtab->check_format(space, format);
 }
 
 static inline void
@@ -472,9 +470,9 @@ space_add_primary_key_xc(struct space *space)
 }
 
 static inline void
-space_check_format_xc(struct space *new_space, struct space *old_space)
+space_check_format_xc(struct space *space, struct tuple_format *format)
 {
-	if (space_check_format(new_space, old_space) != 0)
+	if (space_check_format(space, format) != 0)
 		diag_raise();
 }
 
diff --git a/src/box/sysview_engine.c b/src/box/sysview_engine.c
index 8cfdeeba..bb31edbf 100644
--- a/src/box/sysview_engine.c
+++ b/src/box/sysview_engine.c
@@ -154,10 +154,10 @@ sysview_space_prepare_alter(struct space *old_space, struct space *new_space)
 }
 
 static int
-sysview_space_check_format(struct space *new_space, struct space *old_space)
+sysview_space_check_format(struct space *space, struct tuple_format *format)
 {
-	(void)old_space;
-	(void)new_space;
+	(void)space;
+	(void)format;
 	unreachable();
 	return 0;
 }
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 7db7a60f..cbafa122 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1035,14 +1035,14 @@ vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
 }
 
 static int
-vinyl_space_check_format(struct space *new_space, struct space *old_space)
+vinyl_space_check_format(struct space *space, struct tuple_format *format)
 {
-	(void)new_space;
-	struct vy_env *env = vy_env(old_space->engine);
+	(void)format;
+	struct vy_env *env = vy_env(space->engine);
 	/* @sa vy_prepare_alter_space for checks below. */
-	if (old_space->index_count == 0)
+	if (space->index_count == 0)
 		return 0;
-	struct vy_lsm *pk = vy_lsm(old_space->index[0]);
+	struct vy_lsm *pk = vy_lsm(space->index[0]);
 	if (env->status != VINYL_ONLINE)
 		return 0;
 	if (pk->stat.disk.count.rows == 0 && pk->stat.memory.count.rows == 0)
-- 
2.11.0

Re: [PATCH 07/12] space: pass new format instead of new space to space_vtab::check_format

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> The check_format method is supposed to check that all tuples stored in
> an altered space conform to the new format. Let's change its signature
> accordingly.

OK to push.


-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 08/12] alter: introduce preparation phase

[Vladimir Davydov]

We may yield while rebuilding a vinyl index hence we must not modify
the old space before we are done building all indexes, otherwise the
user might see an inconsistent state while a space is being altered.
Currently, this requirement doesn't hold - we both modify the old space
and build the new indexes in AlterSpaceOp::alter. Let's introduce
AlterSpaceOp::prepare method that will perform all yielding operations
and forbid yielding in AlterSpaceOp::alter.

Actually, we introduce two new methods AlterSpaceOp::prepare and abort.
The latter is called if preparation phase failed and we need to undo the
effect of AlterSpaceOp::prepare for all operations that have performed
it. AlterSpaceOp::rollback is now only called in case of WAL error.
AlterSpaceOp::alter may not fail anymore, not that it needs to anyway.

Here's the list of AlterSpaceOp descendants that now use 'prepare'
instead of 'alter': CheckSpaceFormat, CreateIndex, DropIndex, and
RebuildIndex.

While we are at it, let's also add some comments to AlterSpaceOp
methods.
---
 src/box/alter.cc | 104 ++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 88 insertions(+), 16 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index 9dd8d8d5..f5996850 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -614,12 +614,48 @@ struct alter_space;
 class AlterSpaceOp {
 public:
 	AlterSpaceOp(struct alter_space *alter);
+
+	/** Link in alter_space::ops. */
 	struct rlist link;
+	/**
+	 * Called before creating the new space. Used to update
+	 * the space definition and/or key list that will be used
+	 * for creating the new space. Must not yield or fail.
+	 */
 	virtual void alter_def(struct alter_space * /* alter */) {}
+	/**
+	 * Called after creating a new space. Used for performing
+	 * long-lasting operations, such as index rebuild or format
+	 * check. May yield. May throw an exception. Must not modify
+	 * the old space.
+	 */
+	virtual void prepare(struct alter_space * /* alter */) {}
+	/**
+	 * Called if the preparation phase failed after this
+	 * operation has been successfully prepared. Supposed
+	 * to undo the effect of AlterSpaceOp::prepare.
+	 */
+	virtual void abort(struct alter_space * /* alter */) {}
+	/**
+	 * Called after all registered operations have completed
+	 * the preparation phase. Used to propagate the old space
+	 * state to the new space (e.g. move unchanged indexes).
+	 * Must not yield or fail.
+	 */
 	virtual void alter(struct alter_space * /* alter */) {}
+	/**
+	 * Called after the change has been successfully written
+	 * to WAL. Must not fail.
+	 */
 	virtual void commit(struct alter_space * /* alter */,
 			    int64_t /* signature */) {}
+	/**
+	 * Called in case a WAL error occurred. It is supposed to undo
+	 * the effect of AlterSpaceOp::prepare and AlterSpaceOp::alter.
+	 * Must not fail.
+	 */
 	virtual void rollback(struct alter_space * /* alter */) {}
+
 	virtual ~AlterSpaceOp() {}
 
 	void *operator new(size_t size)
@@ -657,6 +693,12 @@ struct alter_space {
 	/** New space. */
 	struct space *new_space;
 	/**
+	 * Space used as the source when building a new index.
+	 * Initially it is set to old_space, but may be reset
+	 * to new_space if the primary key is recreated.
+	 */
+	struct space *source_space;
+	/**
 	 * Assigned to the new primary key definition if we're
 	 * rebuilding the primary key, i.e. changing its key parts
 	 * substantially.
@@ -677,6 +719,7 @@ alter_space_new(struct space *old_space)
 		region_calloc_object_xc(&fiber()->gc, struct alter_space);
 	rlist_create(&alter->ops);
 	alter->old_space = old_space;
+	alter->source_space = old_space;
 	alter->space_def = space_def_dup_xc(alter->old_space->def);
 	if (old_space->format != NULL)
 		alter->new_min_field_count = old_space->format->min_field_count;
@@ -836,12 +879,12 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 	       sizeof(alter->old_space->access));
 
 	/*
-	 * Change the new space: build the new index, rename,
-	 * change the fixed field count.
+	 * Build new indexes, check if tuples conform to
+	 * the new space format.
 	 */
 	try {
 		rlist_foreach_entry(op, &alter->ops, link)
-			op->alter(alter);
+			op->prepare(alter);
 	} catch (Exception *e) {
 		/*
 		 * Undo space changes from the last successful
@@ -853,7 +896,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 		while (op != rlist_first_entry(&alter->ops,
 					       class AlterSpaceOp, link)) {
 			op = rlist_prev_entry(op, link);
-			op->rollback(alter);
+			op->abort(alter);
 		}
 		throw;
 	}
@@ -863,6 +906,10 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
 	 * this point.
 	 */
 
+	/* Move old indexes, update space format. */
+	rlist_foreach_entry(op, &alter->ops, link)
+		op->alter(alter);
+
 	/* Rebuild index maps once for all indexes. */
 	space_fill_index_map(alter->old_space);
 	space_fill_index_map(alter->new_space);
@@ -900,11 +947,11 @@ class CheckSpaceFormat: public AlterSpaceOp
 public:
 	CheckSpaceFormat(struct alter_space *alter)
 		:AlterSpaceOp(alter) {}
-	virtual void alter(struct alter_space *alter);
+	virtual void prepare(struct alter_space *alter);
 };
 
 void
-CheckSpaceFormat::alter(struct alter_space *alter)
+CheckSpaceFormat::prepare(struct alter_space *alter)
 {
 	struct space *new_space = alter->new_space;
 	struct space *old_space = alter->old_space;
@@ -987,7 +1034,7 @@ public:
 	/** A reference to the definition of the dropped index. */
 	struct index_def *old_index_def;
 	virtual void alter_def(struct alter_space *alter);
-	virtual void alter(struct alter_space *alter);
+	virtual void prepare(struct alter_space *alter);
 	virtual void commit(struct alter_space *alter, int64_t lsn);
 };
 
@@ -1003,7 +1050,7 @@ DropIndex::alter_def(struct alter_space * /* alter */)
 
 /* Do the drop. */
 void
-DropIndex::alter(struct alter_space *alter)
+DropIndex::prepare(struct alter_space *alter)
 {
 	if (old_index_def->iid == 0)
 		space_drop_primary_key(alter->new_space);
@@ -1151,7 +1198,8 @@ public:
 	/** New index index_def. */
 	struct index_def *new_index_def;
 	virtual void alter_def(struct alter_space *alter);
-	virtual void alter(struct alter_space *alter);
+	virtual void prepare(struct alter_space *alter);
+	virtual void abort(struct alter_space *alter);
 	virtual void commit(struct alter_space *alter, int64_t lsn);
 	virtual void rollback(struct alter_space *alter);
 	virtual ~CreateIndex();
@@ -1175,7 +1223,7 @@ CreateIndex::alter_def(struct alter_space *alter)
  * they are fully enabled at all times.
  */
 void
-CreateIndex::alter(struct alter_space *alter)
+CreateIndex::prepare(struct alter_space *alter)
 {
 	if (new_index_def->iid == 0) {
 		/*
@@ -1189,6 +1237,12 @@ CreateIndex::alter(struct alter_space *alter)
 		 * all keys.
 		 */
 		space_add_primary_key_xc(alter->new_space);
+		/**
+		 * The primary index is recreated hence the
+		 * old data (if any) is discarded. Use the
+		 * new space for building secondary indexes.
+		 */
+		alter->source_space = alter->new_space;
 		return;
 	}
 	/**
@@ -1197,11 +1251,20 @@ CreateIndex::alter(struct alter_space *alter)
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_index_xc(alter->new_space, new_index,
+	space_build_index_xc(alter->source_space, new_index,
 			     alter->new_space->format);
 }
 
 void
+CreateIndex::abort(struct alter_space *alter)
+{
+	struct index *new_index = space_index(alter->new_space,
+					      new_index_def->iid);
+	assert(new_index != NULL);
+	index_abort_create(new_index);
+}
+
+void
 CreateIndex::commit(struct alter_space *alter, int64_t signature)
 {
 	struct index *new_index = space_index(alter->new_space,
@@ -1249,7 +1312,8 @@ public:
 	/** Old index index_def. */
 	struct index_def *old_index_def;
 	virtual void alter_def(struct alter_space *alter);
-	virtual void alter(struct alter_space *alter);
+	virtual void prepare(struct alter_space *alter);
+	virtual void abort(struct alter_space *alter);
 	virtual void commit(struct alter_space *alter, int64_t signature);
 	virtual void rollback(struct alter_space *alter);
 	virtual ~RebuildIndex();
@@ -1264,15 +1328,23 @@ RebuildIndex::alter_def(struct alter_space *alter)
 }
 
 void
-RebuildIndex::alter(struct alter_space *alter)
+RebuildIndex::prepare(struct alter_space *alter)
 {
 	/* Get the new index and build it.  */
 	struct index *new_index = space_index(alter->new_space,
 					      new_index_def->iid);
 	assert(new_index != NULL);
-	space_build_index_xc(new_index_def->iid != 0 ?
-			     alter->new_space : alter->old_space,
-			     new_index, alter->new_space->format);
+	space_build_index_xc(alter->source_space, new_index,
+			     alter->new_space->format);
+}
+
+void
+RebuildIndex::abort(struct alter_space *alter)
+{
+	struct index *new_index = space_index(alter->new_space,
+					      new_index_def->iid);
+	assert(new_index != NULL);
+	index_abort_create(new_index);
 }
 
 void
-- 
2.11.0

[tarantool-patches] Re: [PATCH 08/12] alter: introduce preparation phase

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> We may yield while rebuilding a vinyl index hence we must not modify
> the old space before we are done building all indexes, otherwise the
> user might see an inconsistent state while a space is being altered.
> Currently, this requirement doesn't hold - we both modify the old space
> and build the new indexes in AlterSpaceOp::alter. Let's introduce
> AlterSpaceOp::prepare method that will perform all yielding operations
> and forbid yielding in AlterSpaceOp::alter.
> 
> Actually, we introduce two new methods AlterSpaceOp::prepare and abort.
> The latter is called if preparation phase failed and we need to undo the
> effect of AlterSpaceOp::prepare for all operations that have performed
> it. AlterSpaceOp::rollback is now only called in case of WAL error.
> AlterSpaceOp::alter may not fail anymore, not that it needs to anyway.
> 
> Here's the list of AlterSpaceOp descendants that now use 'prepare'
> instead of 'alter': CheckSpaceFormat, CreateIndex, DropIndex, and
> RebuildIndex.

Any part of alter may fail. alter_def may fail. We don't have a
separate abort() callback for a failed alter_def. Undoing effects
of alter in 3 separate calls (abort(), rollback(), destructor) 
creates a lot of confusion. The idea is that each operation should
clean things up automatically: if commit() was called, old stuff
needs to  be cleaned up. If rollback() was called, new stuff needs
to be cleaned up. If neither was called, we assume we didn't reach
the point of writing to WAL, and clean up everything that has been
built up so far.

> While we are at it, let's also add some comments to AlterSpaceOp
> methods.
> ---
>  src/box/alter.cc | 104 ++++++++++++++++++++++++++++++++++++++++++++++---------
>  1 file changed, 88 insertions(+), 16 deletions(-)
> 
> diff --git a/src/box/alter.cc b/src/box/alter.cc
> index 9dd8d8d5..f5996850 100644
> --- a/src/box/alter.cc
> +++ b/src/box/alter.cc
> @@ -614,12 +614,48 @@ struct alter_space;
>  class AlterSpaceOp {
>  public:
>  	AlterSpaceOp(struct alter_space *alter);
> +
> +	/** Link in alter_space::ops. */
>  	struct rlist link;
> +	/**
> +	 * Called before creating the new space. Used to update
> +	 * the space definition and/or key list that will be used
> +	 * for creating the new space. Must not yield or fail.
> +	 */
>  	virtual void alter_def(struct alter_space * /* alter */) {}
> +	/**
> +	 * Called after creating a new space. Used for performing
> +	 * long-lasting operations, such as index rebuild or format
> +	 * check. May yield. May throw an exception. Must not modify
> +	 * the old space.
> +	 */
> +	virtual void prepare(struct alter_space * /* alter */) {}
> +	/**
> +	 * Called if the preparation phase failed after this
> +	 * operation has been successfully prepared. Supposed
> +	 * to undo the effect of AlterSpaceOp::prepare.
> +	 */
> +	virtual void abort(struct alter_space * /* alter */) {}
> +	/**
> +	 * Called after all registered operations have completed
> +	 * the preparation phase. Used to propagate the old space
> +	 * state to the new space (e.g. move unchanged indexes).
> +	 * Must not yield or fail.
> +	 */
>  	virtual void alter(struct alter_space * /* alter */) {}
> +	/**
> +	 * Called after the change has been successfully written
> +	 * to WAL. Must not fail.
> +	 */
>  	virtual void commit(struct alter_space * /* alter */,
>  			    int64_t /* signature */) {}
> +	/**
> +	 * Called in case a WAL error occurred. It is supposed to undo
> +	 * the effect of AlterSpaceOp::prepare and AlterSpaceOp::alter.
> +	 * Must not fail.
> +	 */
>  	virtual void rollback(struct alter_space * /* alter */) {}
> +
>  	virtual ~AlterSpaceOp() {}
>  
>  	void *operator new(size_t size)
> @@ -657,6 +693,12 @@ struct alter_space {
>  	/** New space. */
>  	struct space *new_space;
>  	/**
> +	 * Space used as the source when building a new index.
> +	 * Initially it is set to old_space, but may be reset
> +	 * to new_space if the primary key is recreated.
> +	 */
> +	struct space *source_space;

Sounds like this shouldn't be in alter, but should be in its own
op designated to recreation of the primary key.

> +	/**
>  	 * Assigned to the new primary key definition if we're
>  	 * rebuilding the primary key, i.e. changing its key parts
>  	 * substantially.
> @@ -677,6 +719,7 @@ alter_space_new(struct space *old_space)
>  		region_calloc_object_xc(&fiber()->gc, struct alter_space);
>  	rlist_create(&alter->ops);
>  	alter->old_space = old_space;
> +	alter->source_space = old_space;
>  	alter->space_def = space_def_dup_xc(alter->old_space->def);
>  	if (old_space->format != NULL)
>  		alter->new_min_field_count = old_space->format->min_field_count;
> @@ -836,12 +879,12 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
>  	       sizeof(alter->old_space->access));
>  
>  	/*
> -	 * Change the new space: build the new index, rename,
> -	 * change the fixed field count.
> +	 * Build new indexes, check if tuples conform to
> +	 * the new space format.
>  	 */
>  	try {
>  		rlist_foreach_entry(op, &alter->ops, link)
> -			op->alter(alter);
> +			op->prepare(alter);
>  	} catch (Exception *e) {
>  		/*
>  		 * Undo space changes from the last successful
> @@ -853,7 +896,7 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
>  		while (op != rlist_first_entry(&alter->ops,
>  					       class AlterSpaceOp, link)) {
>  			op = rlist_prev_entry(op, link);
> -			op->rollback(alter);
> +			op->abort(alter);
>  		}
>  		throw;
>  	}
> @@ -863,6 +906,10 @@ alter_space_do(struct txn *txn, struct alter_space *alter)
>  	 * this point.
>  	 */
>  
> +	/* Move old indexes, update space format. */
> +	rlist_foreach_entry(op, &alter->ops, link)
> +		op->alter(alter);
> +
>  	/* Rebuild index maps once for all indexes. */
>  	space_fill_index_map(alter->old_space);
>  	space_fill_index_map(alter->new_space);
> @@ -900,11 +947,11 @@ class CheckSpaceFormat: public AlterSpaceOp
>  public:
>  	CheckSpaceFormat(struct alter_space *alter)
>  		:AlterSpaceOp(alter) {}
> -	virtual void alter(struct alter_space *alter);
> +	virtual void prepare(struct alter_space *alter);
>  };
>  
>  void
> -CheckSpaceFormat::alter(struct alter_space *alter)
> +CheckSpaceFormat::prepare(struct alter_space *alter)
>  {
>  	struct space *new_space = alter->new_space;
>  	struct space *old_space = alter->old_space;
> @@ -987,7 +1034,7 @@ public:
>  	/** A reference to the definition of the dropped index. */
>  	struct index_def *old_index_def;
>  	virtual void alter_def(struct alter_space *alter);
> -	virtual void alter(struct alter_space *alter);
> +	virtual void prepare(struct alter_space *alter);
>  	virtual void commit(struct alter_space *alter, int64_t lsn);
>  };
>  
> @@ -1003,7 +1050,7 @@ DropIndex::alter_def(struct alter_space * /* alter */)
>  
>  /* Do the drop. */
>  void
> -DropIndex::alter(struct alter_space *alter)
> +DropIndex::prepare(struct alter_space *alter)
>  {
>  	if (old_index_def->iid == 0)
>  		space_drop_primary_key(alter->new_space);
> @@ -1151,7 +1198,8 @@ public:
>  	/** New index index_def. */
>  	struct index_def *new_index_def;
>  	virtual void alter_def(struct alter_space *alter);
> -	virtual void alter(struct alter_space *alter);
> +	virtual void prepare(struct alter_space *alter);
> +	virtual void abort(struct alter_space *alter);
>  	virtual void commit(struct alter_space *alter, int64_t lsn);
>  	virtual void rollback(struct alter_space *alter);
>  	virtual ~CreateIndex();
> @@ -1175,7 +1223,7 @@ CreateIndex::alter_def(struct alter_space *alter)
>   * they are fully enabled at all times.
>   */
>  void
> -CreateIndex::alter(struct alter_space *alter)
> +CreateIndex::prepare(struct alter_space *alter)
>  {
>  	if (new_index_def->iid == 0) {
>  		/*
> @@ -1189,6 +1237,12 @@ CreateIndex::alter(struct alter_space *alter)
>  		 * all keys.
>  		 */
>  		space_add_primary_key_xc(alter->new_space);
> +		/**
> +		 * The primary index is recreated hence the
> +		 * old data (if any) is discarded. Use the
> +		 * new space for building secondary indexes.
> +		 */
> +		alter->source_space = alter->new_space;
>  		return;
>  	}
>  	/**
> @@ -1197,11 +1251,20 @@ CreateIndex::alter(struct alter_space *alter)
>  	struct index *new_index = space_index(alter->new_space,
>  					      new_index_def->iid);
>  	assert(new_index != NULL);
> -	space_build_index_xc(alter->new_space, new_index,
> +	space_build_index_xc(alter->source_space, new_index,
>  			     alter->new_space->format);
>  }
>  
>  void
> +CreateIndex::abort(struct alter_space *alter)
> +{
> +	struct index *new_index = space_index(alter->new_space,
> +					      new_index_def->iid);
> +	assert(new_index != NULL);
> +	index_abort_create(new_index);
> +}
> +
> +void
>  CreateIndex::commit(struct alter_space *alter, int64_t signature)
>  {
>  	struct index *new_index = space_index(alter->new_space,
> @@ -1249,7 +1312,8 @@ public:
>  	/** Old index index_def. */
>  	struct index_def *old_index_def;
>  	virtual void alter_def(struct alter_space *alter);
> -	virtual void alter(struct alter_space *alter);
> +	virtual void prepare(struct alter_space *alter);
> +	virtual void abort(struct alter_space *alter);
>  	virtual void commit(struct alter_space *alter, int64_t signature);
>  	virtual void rollback(struct alter_space *alter);
>  	virtual ~RebuildIndex();
> @@ -1264,15 +1328,23 @@ RebuildIndex::alter_def(struct alter_space *alter)
>  }
>  
>  void
> -RebuildIndex::alter(struct alter_space *alter)
> +RebuildIndex::prepare(struct alter_space *alter)
>  {
>  	/* Get the new index and build it.  */
>  	struct index *new_index = space_index(alter->new_space,
>  					      new_index_def->iid);
>  	assert(new_index != NULL);
> -	space_build_index_xc(new_index_def->iid != 0 ?
> -			     alter->new_space : alter->old_space,
> -			     new_index, alter->new_space->format);
> +	space_build_index_xc(alter->source_space, new_index,
> +			     alter->new_space->format);
> +}
> +
> +void
> +RebuildIndex::abort(struct alter_space *alter)
> +{
> +	struct index *new_index = space_index(alter->new_space,
> +					      new_index_def->iid);
> +	assert(new_index != NULL);
> +	index_abort_create(new_index);
>  }
>  

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

Re: [tarantool-patches] Re: [PATCH 08/12] alter: introduce preparation phase

[Vladimir Davydov]

On Mon, Apr 09, 2018 at 11:46:47PM +0300, Konstantin Osipov wrote:
> * Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> > We may yield while rebuilding a vinyl index hence we must not modify
> > the old space before we are done building all indexes, otherwise the
> > user might see an inconsistent state while a space is being altered.
> > Currently, this requirement doesn't hold - we both modify the old space
> > and build the new indexes in AlterSpaceOp::alter. Let's introduce
> > AlterSpaceOp::prepare method that will perform all yielding operations
> > and forbid yielding in AlterSpaceOp::alter.
> > 
> > Actually, we introduce two new methods AlterSpaceOp::prepare and abort.
> > The latter is called if preparation phase failed and we need to undo the
> > effect of AlterSpaceOp::prepare for all operations that have performed
> > it. AlterSpaceOp::rollback is now only called in case of WAL error.
> > AlterSpaceOp::alter may not fail anymore, not that it needs to anyway.
> > 
> > Here's the list of AlterSpaceOp descendants that now use 'prepare'
> > instead of 'alter': CheckSpaceFormat, CreateIndex, DropIndex, and
> > RebuildIndex.
> 
> Any part of alter may fail. alter_def may fail.

I changed that - see the new comments. Now 'alter_def' may not fail, nor
may 'alter'. Only 'prepare' is allowed to fail or yield. This simplifies
the code of alter_space_do() and makes it easier to follow IMO. Anyway,
we don't really need to fail while performing these operations, not with
the new 'prepare' phase.

> We don't have a
> separate abort() callback for a failed alter_def. Undoing effects
> of alter in 3 separate calls (abort(), rollback(), destructor) 
> creates a lot of confusion. The idea is that each operation should
> clean things up automatically: if commit() was called, old stuff
> needs to  be cleaned up. If rollback() was called, new stuff needs
> to be cleaned up. If neither was called, we assume we didn't reach
> the point of writing to WAL, and clean up everything that has been
> built up so far.

The problem is that we first call 'prepare' for all registered
operations. If 'prepare' fails for any of those operations, we
need to undo its effect for all operations for which 'prepare'
succeeded. I see the following ways of doing this:

 - Reuse 'rollback' for undoing the effect of 'prepare', but this way we
   would have to be able to differentiate between 'rollback' after WAL
   error and 'rollback' after 'prepare' for all operations, even those
   that don't use 'prepare' at all, e.g. ModifySpace. This would look
   convoluted IMO.

 - Undo the effect of 'prepare' in destructor. I don't like it, because
   IMO destructor should be used only for freeing up resources. Calling,
   for example, index_abort_create() from it would look ugly IMO.

 - Introduce 'abort' that would undo the effect of 'prepare'. All things
   considered, I find it the best way to go from the code readability
   point of view.

> 
> > While we are at it, let's also add some comments to AlterSpaceOp
> > methods.
> > ---
> >  src/box/alter.cc | 104 ++++++++++++++++++++++++++++++++++++++++++++++---------
> >  1 file changed, 88 insertions(+), 16 deletions(-)
> > 
> > diff --git a/src/box/alter.cc b/src/box/alter.cc
> > index 9dd8d8d5..f5996850 100644
> > --- a/src/box/alter.cc
> > +++ b/src/box/alter.cc
> > @@ -614,12 +614,48 @@ struct alter_space;
> >  class AlterSpaceOp {
> >  public:
> >  	AlterSpaceOp(struct alter_space *alter);
> > +
> > +	/** Link in alter_space::ops. */
> >  	struct rlist link;
> > +	/**
> > +	 * Called before creating the new space. Used to update
> > +	 * the space definition and/or key list that will be used
> > +	 * for creating the new space. Must not yield or fail.
> > +	 */
> >  	virtual void alter_def(struct alter_space * /* alter */) {}
> > +	/**
> > +	 * Called after creating a new space. Used for performing
> > +	 * long-lasting operations, such as index rebuild or format
> > +	 * check. May yield. May throw an exception. Must not modify
> > +	 * the old space.
> > +	 */
> > +	virtual void prepare(struct alter_space * /* alter */) {}
> > +	/**
> > +	 * Called if the preparation phase failed after this
> > +	 * operation has been successfully prepared. Supposed
> > +	 * to undo the effect of AlterSpaceOp::prepare.
> > +	 */
> > +	virtual void abort(struct alter_space * /* alter */) {}
> > +	/**
> > +	 * Called after all registered operations have completed
> > +	 * the preparation phase. Used to propagate the old space
> > +	 * state to the new space (e.g. move unchanged indexes).
> > +	 * Must not yield or fail.
> > +	 */
> >  	virtual void alter(struct alter_space * /* alter */) {}
> > +	/**
> > +	 * Called after the change has been successfully written
> > +	 * to WAL. Must not fail.
> > +	 */
> >  	virtual void commit(struct alter_space * /* alter */,
> >  			    int64_t /* signature */) {}
> > +	/**
> > +	 * Called in case a WAL error occurred. It is supposed to undo
> > +	 * the effect of AlterSpaceOp::prepare and AlterSpaceOp::alter.
> > +	 * Must not fail.
> > +	 */
> >  	virtual void rollback(struct alter_space * /* alter */) {}
> > +
> >  	virtual ~AlterSpaceOp() {}
> >  
> >  	void *operator new(size_t size)
> > @@ -657,6 +693,12 @@ struct alter_space {
> >  	/** New space. */
> >  	struct space *new_space;
> >  	/**
> > +	 * Space used as the source when building a new index.
> > +	 * Initially it is set to old_space, but may be reset
> > +	 * to new_space if the primary key is recreated.
> > +	 */
> > +	struct space *source_space;
> 
> Sounds like this shouldn't be in alter, but should be in its own
> op designated to recreation of the primary key.

We need to update it for all CreateIndex operations, whenever we detect
that the primary index is truncated, i.e. CreateIndex is called for the
primary index. I haven't figured out a better way to do it. We might
need to discuss it f2f.

Re: [tarantool-patches] Re: [PATCH 08/12] alter: introduce preparation phase

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/10 11:32]:
> The problem is that we first call 'prepare' for all registered
> operations. If 'prepare' fails for any of those operations, we
> need to undo its effect for all operations for which 'prepare'
> succeeded. I see the following ways of doing this:
> 
>  - Reuse 'rollback' for undoing the effect of 'prepare', but this way we
>    would have to be able to differentiate between 'rollback' after WAL
>    error and 'rollback' after 'prepare' for all operations, even those
>    that don't use 'prepare' at all, e.g. ModifySpace. This would look
>    convoluted IMO.
> 
>  - Undo the effect of 'prepare' in destructor. I don't like it, because
>    IMO destructor should be used only for freeing up resources. Calling,
>    for example, index_abort_create() from it would look ugly IMO.

I prefer this option. I don't see why we need two functions simply
because "destructor should be used only for freeing up resources".
Let's drop C++ there altogether and forget this argument, if it
makes it easier.
> 
>  - Introduce 'abort' that would undo the effect of 'prepare'. All things
>    considered, I find it the best way to go from the code readability
>    point of view.
> 
> > 
> > > While we are at it, let's also add some comments to AlterSpaceOp
> > > methods.
> > > ---
> > >  src/box/alter.cc | 104 ++++++++++++++++++++++++++++++++++++++++++++++---------
> > >  1 file changed, 88 insertions(+), 16 deletions(-)
> > > 
> > > diff --git a/src/box/alter.cc b/src/box/alter.cc
> > > index 9dd8d8d5..f5996850 100644
> > > --- a/src/box/alter.cc
> > > +++ b/src/box/alter.cc
> > > @@ -614,12 +614,48 @@ struct alter_space;
> > >  class AlterSpaceOp {
> > >  public:
> > >  	AlterSpaceOp(struct alter_space *alter);
> > > +
> > > +	/** Link in alter_space::ops. */
> > >  	struct rlist link;
> > > +	/**
> > > +	 * Called before creating the new space. Used to update
> > > +	 * the space definition and/or key list that will be used
> > > +	 * for creating the new space. Must not yield or fail.
> > > +	 */
> > >  	virtual void alter_def(struct alter_space * /* alter */) {}
> > > +	/**
> > > +	 * Called after creating a new space. Used for performing
> > > +	 * long-lasting operations, such as index rebuild or format
> > > +	 * check. May yield. May throw an exception. Must not modify
> > > +	 * the old space.
> > > +	 */
> > > +	virtual void prepare(struct alter_space * /* alter */) {}
> > > +	/**
> > > +	 * Called if the preparation phase failed after this
> > > +	 * operation has been successfully prepared. Supposed
> > > +	 * to undo the effect of AlterSpaceOp::prepare.
> > > +	 */
> > > +	virtual void abort(struct alter_space * /* alter */) {}
> > > +	/**
> > > +	 * Called after all registered operations have completed
> > > +	 * the preparation phase. Used to propagate the old space
> > > +	 * state to the new space (e.g. move unchanged indexes).
> > > +	 * Must not yield or fail.
> > > +	 */
> > >  	virtual void alter(struct alter_space * /* alter */) {}
> > > +	/**
> > > +	 * Called after the change has been successfully written
> > > +	 * to WAL. Must not fail.
> > > +	 */
> > >  	virtual void commit(struct alter_space * /* alter */,
> > >  			    int64_t /* signature */) {}
> > > +	/**
> > > +	 * Called in case a WAL error occurred. It is supposed to undo
> > > +	 * the effect of AlterSpaceOp::prepare and AlterSpaceOp::alter.
> > > +	 * Must not fail.
> > > +	 */
> > >  	virtual void rollback(struct alter_space * /* alter */) {}
> > > +
> > >  	virtual ~AlterSpaceOp() {}
> > >  
> > >  	void *operator new(size_t size)
> > > @@ -657,6 +693,12 @@ struct alter_space {
> > >  	/** New space. */
> > >  	struct space *new_space;
> > >  	/**
> > > +	 * Space used as the source when building a new index.
> > > +	 * Initially it is set to old_space, but may be reset
> > > +	 * to new_space if the primary key is recreated.
> > > +	 */
> > > +	struct space *source_space;
> > 
> > Sounds like this shouldn't be in alter, but should be in its own
> > op designated to recreation of the primary key.
> 
> We need to update it for all CreateIndex operations, whenever we detect
> that the primary index is truncated, i.e. CreateIndex is called for the
> primary index. I haven't figured out a better way to do it. We might
> need to discuss it f2f.

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 09/12] alter: zap space_def_check_compatibility

[Vladimir Davydov]

space_def_check_compatibility takes two space definitions, original and
the one created by alter, and checks that they are compatible, i.e.

 - space id and engine do not differ
 - temporary flag is not flipped
 - field definitions are compatible

The last two checks are performed only if the space is empty, which is
indicated by 'is_space_empty' argument.

This function is called by space_vtab::prepare_alter engine callbacks
with is_space_empty set to false and by generic space ALTER code with
is_space_empty set to true. The reason for this is that we do not know
whether the space is empty when ALTER is initiated.

Actually, there's no need at all to assure that field definitions are
compatible: it is guaranteed by space_vtab::check_format callback,
though the error message is a bit different but still clear. After
removing this piece of code from space_def_check_compatibility, this
function doesn't make sense any more: we can fold space id and engine
checks right into on_replace_dd_space where they originally resided;
as for the temporary flag, it can be set only for memtx spaces so we
can move this check to memtx_space_prepare_alter. Let's do this and
remove this weird function.
---
 src/box/alter.cc             | 14 +++++++------
 src/box/memtx_space.c        | 13 ++++++++----
 src/box/space_def.c          | 49 --------------------------------------------
 src/box/space_def.h          | 26 -----------------------
 src/box/vinyl.c              |  3 ---
 test/box/alter.result        | 21 +++++++------------
 test/box/alter_limits.result |  9 +++-----
 7 files changed, 27 insertions(+), 108 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index f5996850..d05c6483 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -1639,12 +1639,14 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event)
 		access_check_ddl(def->name, def->uid, SC_SPACE, PRIV_A, true);
 		auto def_guard =
 			make_scoped_guard([=] { space_def_delete(def); });
-		/*
-		 * Check basic options. Assume the space to be
-		 * empty, because we can not calculate here
-		 * a size of a vinyl space.
-		 */
-		space_def_check_compatibility_xc(old_space->def, def, true);
+		if (def->id != space_id(old_space))
+			tnt_raise(ClientError, ER_ALTER_SPACE,
+				  space_name(old_space),
+				  "space id is immutable");
+		if (strcmp(def->engine_name, old_space->def->engine_name) != 0)
+			tnt_raise(ClientError, ER_ALTER_SPACE,
+				  space_name(old_space),
+				  "can not change space engine");
 		/*
 		 * Allow change of space properties, but do it
 		 * in WAL-error-safe mode.
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index be687288..ae266cc9 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -825,12 +825,17 @@ memtx_space_prepare_alter(struct space *old_space, struct space *new_space)
 {
 	struct memtx_space *old_memtx_space = (struct memtx_space *)old_space;
 	struct memtx_space *new_memtx_space = (struct memtx_space *)new_space;
+
+	if (old_memtx_space->bsize != 0 &&
+	    old_space->def->opts.temporary != new_space->def->opts.temporary) {
+		diag_set(ClientError, ER_ALTER_SPACE, old_space->def->name,
+			 "can not switch temporary flag on a non-empty space");
+		return -1;
+	}
+
 	new_memtx_space->replace = old_memtx_space->replace;
 	new_memtx_space->bsize = old_memtx_space->bsize;
-	bool is_empty = old_space->index_count == 0 ||
-			index_size(old_space->index[0]) == 0;
-	return space_def_check_compatibility(old_space->def,
-					     new_space->def, is_empty);
+	return 0;
 }
 
 /* }}} DDL */
diff --git a/src/box/space_def.c b/src/box/space_def.c
index ecb5ad72..7349c214 100644
--- a/src/box/space_def.c
+++ b/src/box/space_def.c
@@ -142,52 +142,3 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count,
 	}
 	return def;
 }
-
-int
-space_def_check_compatibility(const struct space_def *old_def,
-			      const struct space_def *new_def,
-			      bool is_space_empty)
-{
-	if (strcmp(new_def->engine_name, old_def->engine_name) != 0) {
-		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
-			 "can not change space engine");
-		return -1;
-	}
-	if (new_def->id != old_def->id) {
-		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
-			 "space id is immutable");
-		return -1;
-	}
-	if (is_space_empty)
-		return 0;
-
-	if (new_def->exact_field_count != 0 &&
-	    new_def->exact_field_count != old_def->exact_field_count) {
-		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
-			 "can not change field count on a non-empty space");
-		return -1;
-	}
-	if (new_def->opts.temporary != old_def->opts.temporary) {
-		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
-			 "can not switch temporary flag on a non-empty space");
-		return -1;
-	}
-	uint32_t field_count = MIN(new_def->field_count, old_def->field_count);
-	for (uint32_t i = 0; i < field_count; ++i) {
-		enum field_type old_type = old_def->fields[i].type;
-		enum field_type new_type = new_def->fields[i].type;
-		if (!field_type1_contains_type2(new_type, old_type) &&
-		    !field_type1_contains_type2(old_type, new_type)) {
-			const char *msg =
-				tt_sprintf("Can not change a field type from "\
-					   "%s to %s on a not empty space",
-					   field_type_strs[old_type],
-					   field_type_strs[new_type]);
-			diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
-				 msg);
-			return -1;
-		}
-	}
-	return 0;
-}
-
diff --git a/src/box/space_def.h b/src/box/space_def.h
index 54fe2c71..97c7e138 100644
--- a/src/box/space_def.h
+++ b/src/box/space_def.h
@@ -133,22 +133,6 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count,
 	      const struct space_opts *opts, const struct field_def *fields,
 	      uint32_t field_count);
 
-/**
- * Check that a space with @an old_def can be altered to have
- * @a new_def.
- * @param old_def Old space definition.
- * @param new_def New space definition.
- * @param is_space_empty True, if a space is empty.
- *
- * @retval  0 Space definition can be altered to @a new_def.
- * @retval -1 Client error.
- */
-int
-space_def_check_compatibility(const struct space_def *old_def,
-			      const struct space_def *new_def,
-			      bool is_space_empty);
-
-
 #if defined(__cplusplus)
 } /* extern "C" */
 
@@ -178,16 +162,6 @@ space_def_new_xc(uint32_t id, uint32_t uid, uint32_t exact_field_count,
 	return ret;
 }
 
-static inline void
-space_def_check_compatibility_xc(const struct space_def *old_def,
-				 const struct space_def *new_def,
-				 bool is_space_empty)
-{
-	if (space_def_check_compatibility(old_def, new_def,
-					  is_space_empty) != 0)
-		diag_raise();
-}
-
 #endif /* __cplusplus */
 
 #endif /* TARANTOOL_BOX_SPACE_DEF_H_INCLUDED */
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index cbafa122..0c475af4 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1017,9 +1017,6 @@ vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
 			}
 		}
 	}
-	if (space_def_check_compatibility(old_space->def, new_space->def,
-					  false) != 0)
-		return -1;
 	if (old_space->index_count < new_space->index_count) {
 		diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
 			 "adding an index to a non-empty space");
diff --git a/test/box/alter.result b/test/box/alter.result
index 347de477..5e3cbf63 100644
--- a/test/box/alter.result
+++ b/test/box/alter.result
@@ -1244,8 +1244,7 @@ t[5] = 1
 ...
 box.space._space:replace(t)
 ---
-- error: 'Can''t modify space ''test'': can not change field count on a non-empty
-    space'
+- error: Vinyl does not support changing space format of a non-empty space
 ...
 s:drop()
 ---
@@ -1333,8 +1332,7 @@ ok_format_change(3, 'any')
 -- unsigned --X--> string
 fail_format_change(3, 'string')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from unsigned to string
-  on a not empty space'
+- 'Tuple field 3 type does not match one required by operation: expected string'
 ...
 -- unsigned -----> number
 ok_format_change(3, 'number')
@@ -1351,8 +1349,7 @@ ok_format_change(3, 'scalar')
 -- unsigned --X--> map
 fail_format_change(3, 'map')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from unsigned to map
-  on a not empty space'
+- 'Tuple field 3 type does not match one required by operation: expected map'
 ...
 -- string -----> any
 ok_format_change(4, 'any')
@@ -1365,8 +1362,7 @@ ok_format_change(4, 'scalar')
 -- string --X--> boolean
 fail_format_change(4, 'boolean')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from string to boolean
-  on a not empty space'
+- 'Tuple field 4 type does not match one required by operation: expected boolean'
 ...
 -- number -----> any
 ok_format_change(5, 'any')
@@ -1409,8 +1405,7 @@ ok_format_change(7, 'scalar')
 -- boolean --X--> string
 fail_format_change(7, 'string')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from boolean to string
-  on a not empty space'
+- 'Tuple field 7 type does not match one required by operation: expected string'
 ...
 -- scalar -----> any
 ok_format_change(8, 'any')
@@ -1428,8 +1423,7 @@ ok_format_change(9, 'any')
 -- array --X--> scalar
 fail_format_change(9, 'scalar')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from array to scalar
-  on a not empty space'
+- 'Tuple field 9 type does not match one required by operation: expected scalar'
 ...
 -- map -----> any
 ok_format_change(10, 'any')
@@ -1438,8 +1432,7 @@ ok_format_change(10, 'any')
 -- map --X--> scalar
 fail_format_change(10, 'scalar')
 ---
-- 'Can''t modify space ''test'': Can not change a field type from map to scalar on
-  a not empty space'
+- 'Tuple field 10 type does not match one required by operation: expected scalar'
 ...
 s:drop()
 ---
diff --git a/test/box/alter_limits.result b/test/box/alter_limits.result
index 93e99dbe..4fd80a37 100644
--- a/test/box/alter_limits.result
+++ b/test/box/alter_limits.result
@@ -285,8 +285,7 @@ FIELD_COUNT = 4
 -- increase field_count -- error
 box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 3}})
 ---
-- error: 'Can''t modify space ''test'': can not change field count on a non-empty
-    space'
+- error: Tuple field count 2 does not match space field count 3
 ...
 s:select{}
 ---
@@ -295,8 +294,7 @@ s:select{}
 -- decrease field_count - error
 box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 1}})
 ---
-- error: 'Can''t modify space ''test'': can not change field count on a non-empty
-    space'
+- error: Tuple field count 2 does not match space field count 1
 ...
 -- remove field_count - ok
 _ = box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 0}})
@@ -309,8 +307,7 @@ s:select{}
 -- increase field_count - error
 box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 3}})
 ---
-- error: 'Can''t modify space ''test'': can not change field count on a non-empty
-    space'
+- error: Tuple field count 2 does not match space field count 3
 ...
 s:truncate()
 ---
-- 
2.11.0

Re: [PATCH 09/12] alter: zap space_def_check_compatibility

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> space_def_check_compatibility takes two space definitions, original and
> the one created by alter, and checks that they are compatible, i.e.

IMO it's bikeshed.

I don't like having checks done at prepare() phase though.
Assuming some heavy lifting could be done by some operations in
the list of ops during prepare(), it's better to move all the
trivial checks in a phase that precedes prepare().

>  - space id and engine do not differ
>  - temporary flag is not flipped
>  - field definitions are compatible
> 
> The last two checks are performed only if the space is empty, which is
> indicated by 'is_space_empty' argument.
> 
> This function is called by space_vtab::prepare_alter engine callbacks
> with is_space_empty set to false and by generic space ALTER code with
> is_space_empty set to true. The reason for this is that we do not know
> whether the space is empty when ALTER is initiated.
> 
> Actually, there's no need at all to assure that field definitions are
> compatible: it is guaranteed by space_vtab::check_format callback,
> though the error message is a bit different but still clear. After
> removing this piece of code from space_def_check_compatibility, this
> function doesn't make sense any more: we can fold space id and engine
> checks right into on_replace_dd_space where they originally resided;
> as for the temporary flag, it can be set only for memtx spaces so we
> can move this check to memtx_space_prepare_alter. Let's do this and
> remove this weird function.
> ---
>  src/box/alter.cc             | 14 +++++++------
>  src/box/memtx_space.c        | 13 ++++++++----
>  src/box/space_def.c          | 49 --------------------------------------------
>  src/box/space_def.h          | 26 -----------------------
>  src/box/vinyl.c              |  3 ---
>  test/box/alter.result        | 21 +++++++------------
>  test/box/alter_limits.result |  9 +++-----
>  7 files changed, 27 insertions(+), 108 deletions(-)
> 
> diff --git a/src/box/alter.cc b/src/box/alter.cc
> index f5996850..d05c6483 100644
> --- a/src/box/alter.cc
> +++ b/src/box/alter.cc
> @@ -1639,12 +1639,14 @@ on_replace_dd_space(struct trigger * /* trigger */, void *event)
>  		access_check_ddl(def->name, def->uid, SC_SPACE, PRIV_A, true);
>  		auto def_guard =
>  			make_scoped_guard([=] { space_def_delete(def); });
> -		/*
> -		 * Check basic options. Assume the space to be
> -		 * empty, because we can not calculate here
> -		 * a size of a vinyl space.
> -		 */
> -		space_def_check_compatibility_xc(old_space->def, def, true);
> +		if (def->id != space_id(old_space))
> +			tnt_raise(ClientError, ER_ALTER_SPACE,
> +				  space_name(old_space),
> +				  "space id is immutable");
> +		if (strcmp(def->engine_name, old_space->def->engine_name) != 0)
> +			tnt_raise(ClientError, ER_ALTER_SPACE,
> +				  space_name(old_space),
> +				  "can not change space engine");
>  		/*
>  		 * Allow change of space properties, but do it
>  		 * in WAL-error-safe mode.
> diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
> index be687288..ae266cc9 100644
> --- a/src/box/memtx_space.c
> +++ b/src/box/memtx_space.c
> @@ -825,12 +825,17 @@ memtx_space_prepare_alter(struct space *old_space, struct space *new_space)
>  {
>  	struct memtx_space *old_memtx_space = (struct memtx_space *)old_space;
>  	struct memtx_space *new_memtx_space = (struct memtx_space *)new_space;
> +
> +	if (old_memtx_space->bsize != 0 &&
> +	    old_space->def->opts.temporary != new_space->def->opts.temporary) {
> +		diag_set(ClientError, ER_ALTER_SPACE, old_space->def->name,
> +			 "can not switch temporary flag on a non-empty space");
> +		return -1;
> +	}
> +
>  	new_memtx_space->replace = old_memtx_space->replace;
>  	new_memtx_space->bsize = old_memtx_space->bsize;
> -	bool is_empty = old_space->index_count == 0 ||
> -			index_size(old_space->index[0]) == 0;
> -	return space_def_check_compatibility(old_space->def,
> -					     new_space->def, is_empty);
> +	return 0;
>  }
>  
>  /* }}} DDL */
> diff --git a/src/box/space_def.c b/src/box/space_def.c
> index ecb5ad72..7349c214 100644
> --- a/src/box/space_def.c
> +++ b/src/box/space_def.c
> @@ -142,52 +142,3 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count,
>  	}
>  	return def;
>  }
> -
> -int
> -space_def_check_compatibility(const struct space_def *old_def,
> -			      const struct space_def *new_def,
> -			      bool is_space_empty)
> -{
> -	if (strcmp(new_def->engine_name, old_def->engine_name) != 0) {
> -		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
> -			 "can not change space engine");
> -		return -1;
> -	}
> -	if (new_def->id != old_def->id) {
> -		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
> -			 "space id is immutable");
> -		return -1;
> -	}
> -	if (is_space_empty)
> -		return 0;
> -
> -	if (new_def->exact_field_count != 0 &&
> -	    new_def->exact_field_count != old_def->exact_field_count) {
> -		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
> -			 "can not change field count on a non-empty space");
> -		return -1;
> -	}
> -	if (new_def->opts.temporary != old_def->opts.temporary) {
> -		diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
> -			 "can not switch temporary flag on a non-empty space");
> -		return -1;
> -	}
> -	uint32_t field_count = MIN(new_def->field_count, old_def->field_count);
> -	for (uint32_t i = 0; i < field_count; ++i) {
> -		enum field_type old_type = old_def->fields[i].type;
> -		enum field_type new_type = new_def->fields[i].type;
> -		if (!field_type1_contains_type2(new_type, old_type) &&
> -		    !field_type1_contains_type2(old_type, new_type)) {
> -			const char *msg =
> -				tt_sprintf("Can not change a field type from "\
> -					   "%s to %s on a not empty space",
> -					   field_type_strs[old_type],
> -					   field_type_strs[new_type]);
> -			diag_set(ClientError, ER_ALTER_SPACE, old_def->name,
> -				 msg);
> -			return -1;
> -		}
> -	}
> -	return 0;
> -}
> -
> diff --git a/src/box/space_def.h b/src/box/space_def.h
> index 54fe2c71..97c7e138 100644
> --- a/src/box/space_def.h
> +++ b/src/box/space_def.h
> @@ -133,22 +133,6 @@ space_def_new(uint32_t id, uint32_t uid, uint32_t exact_field_count,
>  	      const struct space_opts *opts, const struct field_def *fields,
>  	      uint32_t field_count);
>  
> -/**
> - * Check that a space with @an old_def can be altered to have
> - * @a new_def.
> - * @param old_def Old space definition.
> - * @param new_def New space definition.
> - * @param is_space_empty True, if a space is empty.
> - *
> - * @retval  0 Space definition can be altered to @a new_def.
> - * @retval -1 Client error.
> - */
> -int
> -space_def_check_compatibility(const struct space_def *old_def,
> -			      const struct space_def *new_def,
> -			      bool is_space_empty);
> -
> -
>  #if defined(__cplusplus)
>  } /* extern "C" */
>  
> @@ -178,16 +162,6 @@ space_def_new_xc(uint32_t id, uint32_t uid, uint32_t exact_field_count,
>  	return ret;
>  }
>  
> -static inline void
> -space_def_check_compatibility_xc(const struct space_def *old_def,
> -				 const struct space_def *new_def,
> -				 bool is_space_empty)
> -{
> -	if (space_def_check_compatibility(old_def, new_def,
> -					  is_space_empty) != 0)
> -		diag_raise();
> -}
> -
>  #endif /* __cplusplus */
>  
>  #endif /* TARANTOOL_BOX_SPACE_DEF_H_INCLUDED */
> diff --git a/src/box/vinyl.c b/src/box/vinyl.c
> index cbafa122..0c475af4 100644
> --- a/src/box/vinyl.c
> +++ b/src/box/vinyl.c
> @@ -1017,9 +1017,6 @@ vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
>  			}
>  		}
>  	}
> -	if (space_def_check_compatibility(old_space->def, new_space->def,
> -					  false) != 0)
> -		return -1;
>  	if (old_space->index_count < new_space->index_count) {
>  		diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
>  			 "adding an index to a non-empty space");
> diff --git a/test/box/alter.result b/test/box/alter.result
> index 347de477..5e3cbf63 100644
> --- a/test/box/alter.result
> +++ b/test/box/alter.result
> @@ -1244,8 +1244,7 @@ t[5] = 1
>  ...
>  box.space._space:replace(t)
>  ---
> -- error: 'Can''t modify space ''test'': can not change field count on a non-empty
> -    space'
> +- error: Vinyl does not support changing space format of a non-empty space
>  ...
>  s:drop()
>  ---
> @@ -1333,8 +1332,7 @@ ok_format_change(3, 'any')
>  -- unsigned --X--> string
>  fail_format_change(3, 'string')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from unsigned to string
> -  on a not empty space'
> +- 'Tuple field 3 type does not match one required by operation: expected string'
>  ...
>  -- unsigned -----> number
>  ok_format_change(3, 'number')
> @@ -1351,8 +1349,7 @@ ok_format_change(3, 'scalar')
>  -- unsigned --X--> map
>  fail_format_change(3, 'map')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from unsigned to map
> -  on a not empty space'
> +- 'Tuple field 3 type does not match one required by operation: expected map'
>  ...
>  -- string -----> any
>  ok_format_change(4, 'any')
> @@ -1365,8 +1362,7 @@ ok_format_change(4, 'scalar')
>  -- string --X--> boolean
>  fail_format_change(4, 'boolean')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from string to boolean
> -  on a not empty space'
> +- 'Tuple field 4 type does not match one required by operation: expected boolean'
>  ...
>  -- number -----> any
>  ok_format_change(5, 'any')
> @@ -1409,8 +1405,7 @@ ok_format_change(7, 'scalar')
>  -- boolean --X--> string
>  fail_format_change(7, 'string')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from boolean to string
> -  on a not empty space'
> +- 'Tuple field 7 type does not match one required by operation: expected string'
>  ...
>  -- scalar -----> any
>  ok_format_change(8, 'any')
> @@ -1428,8 +1423,7 @@ ok_format_change(9, 'any')
>  -- array --X--> scalar
>  fail_format_change(9, 'scalar')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from array to scalar
> -  on a not empty space'
> +- 'Tuple field 9 type does not match one required by operation: expected scalar'
>  ...
>  -- map -----> any
>  ok_format_change(10, 'any')
> @@ -1438,8 +1432,7 @@ ok_format_change(10, 'any')
>  -- map --X--> scalar
>  fail_format_change(10, 'scalar')
>  ---
> -- 'Can''t modify space ''test'': Can not change a field type from map to scalar on
> -  a not empty space'
> +- 'Tuple field 10 type does not match one required by operation: expected scalar'
>  ...
>  s:drop()
>  ---
> diff --git a/test/box/alter_limits.result b/test/box/alter_limits.result
> index 93e99dbe..4fd80a37 100644
> --- a/test/box/alter_limits.result
> +++ b/test/box/alter_limits.result
> @@ -285,8 +285,7 @@ FIELD_COUNT = 4
>  -- increase field_count -- error
>  box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 3}})
>  ---
> -- error: 'Can''t modify space ''test'': can not change field count on a non-empty
> -    space'
> +- error: Tuple field count 2 does not match space field count 3
>  ...
>  s:select{}
>  ---
> @@ -295,8 +294,7 @@ s:select{}
>  -- decrease field_count - error
>  box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 1}})
>  ---
> -- error: 'Can''t modify space ''test'': can not change field count on a non-empty
> -    space'
> +- error: Tuple field count 2 does not match space field count 1
>  ...
>  -- remove field_count - ok
>  _ = box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 0}})
> @@ -309,8 +307,7 @@ s:select{}
>  -- increase field_count - error
>  box.space['_space']:update(s.id, {{"=", FIELD_COUNT + 1, 3}})
>  ---
> -- error: 'Can''t modify space ''test'': can not change field count on a non-empty
> -    space'
> +- error: Tuple field count 2 does not match space field count 3
>  ...
>  s:truncate()
>  ---
> -- 
> 2.11.0
> 

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 10/12] vinyl: remove superfluous ddl checks

[Vladimir Davydov]

There's no need to iterate over all indexes in vinyl_space_prepare_alter
and check if any of them is going to be rebuilt - alter_space_do already
does that for us. All we need to do is raise an error if the build_index
callback is invoked for a non-empty space after recovery is complete.
The tuple format check is pointless too, as it is already done by
CheckSpaceFormat AlterSpaceOp. Remove them.
---
 src/box/vinyl.c       | 73 ++++++++++++++++-----------------------------------
 test/box/alter.result |  4 +--
 test/vinyl/ddl.result | 26 +++++++++---------
 test/vinyl/gh.result  |  2 +-
 4 files changed, 38 insertions(+), 67 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 0c475af4..47804ecc 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -976,58 +976,12 @@ vinyl_init_system_space(struct space *space)
 static int
 vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
 {
+	(void)new_space;
 	struct vy_env *env = vy_env(old_space->engine);
 
 	if (vinyl_check_wal(env, "DDL") != 0)
 		return -1;
-	/*
-	 * The space with no indexes can contain no rows.
-	 * Allow alter.
-	 */
-	if (old_space->index_count == 0)
-		return 0;
-	struct vy_lsm *pk = vy_lsm(old_space->index[0]);
-	/*
-	 * During WAL recovery, the space may be not empty. But we
-	 * open existing indexes, not creating new ones. Allow
-	 * alter.
-	 */
-	if (env->status != VINYL_ONLINE)
-		return 0;
-	/* The space is empty. Allow alter. */
-	if (pk->stat.disk.count.rows == 0 &&
-	    pk->stat.memory.count.rows == 0)
-		return 0;
-	if (old_space->index_count == new_space->index_count) {
-		/* Check index_defs to be unchanged. */
-		for (uint32_t i = 0; i < old_space->index_count; ++i) {
-			struct index_def *old_def, *new_def;
-			old_def = space_index_def(old_space, i);
-			new_def = space_index_def(new_space, i);
-			/*
-			 * We do not support a full rebuild in
-			 * vinyl yet.
-			 */
-			if (index_def_change_requires_rebuild(old_def,
-							      new_def)) {
-				diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
-					 "changing the definition of "
-					 "a non-empty index");
-				return -1;
-			}
-		}
-	}
-	if (old_space->index_count < new_space->index_count) {
-		diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
-			 "adding an index to a non-empty space");
-		return -1;
-	}
-	if (! tuple_format1_can_store_format2_tuples(new_space->format,
-						     old_space->format)) {
-		diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
-			 "changing space format of a non-empty space");
-		return -1;
-	}
+
 	return 0;
 }
 
@@ -1036,7 +990,6 @@ vinyl_space_check_format(struct space *space, struct tuple_format *format)
 {
 	(void)format;
 	struct vy_env *env = vy_env(space->engine);
-	/* @sa vy_prepare_alter_space for checks below. */
 	if (space->index_count == 0)
 		return 0;
 	struct vy_lsm *pk = vy_lsm(space->index[0]);
@@ -1045,7 +998,7 @@ vinyl_space_check_format(struct space *space, struct tuple_format *format)
 	if (pk->stat.disk.count.rows == 0 && pk->stat.memory.count.rows == 0)
 		return 0;
 	diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
-		 "adding new fields to a non-empty space");
+		 "changing format of a non-empty space");
 	return -1;
 }
 
@@ -1095,6 +1048,24 @@ vinyl_space_build_index(struct space *space, struct index *index,
 			struct tuple_format *format)
 {
 	(void)format;
+
+	struct vy_env *env = vy_env(space->engine);
+	struct vy_lsm *pk = vy_lsm(space->index[0]);
+
+	/*
+	 * During local recovery we are loading existing indexes
+	 * from disk, not building new ones.
+	 */
+	if (env->status != VINYL_INITIAL_RECOVERY_LOCAL &&
+	    env->status != VINYL_FINAL_RECOVERY_LOCAL) {
+		if (pk->stat.disk.count.rows != 0 ||
+		    pk->stat.memory.count.rows != 0) {
+			diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
+				 "building an index for a non-empty space");
+			return -1;
+		}
+	}
+
 	/*
 	 * Unlike Memtx, Vinyl does not need building of a secondary index.
 	 * This is true because of two things:
@@ -1114,7 +1085,7 @@ vinyl_space_build_index(struct space *space, struct index *index,
 		return -1;
 
 	/* Set pointer to the primary key for the new index. */
-	vy_lsm_update_pk(vy_lsm(index), vy_lsm(space_index(space, 0)));
+	vy_lsm_update_pk(vy_lsm(index), pk);
 	return 0;
 }
 
diff --git a/test/box/alter.result b/test/box/alter.result
index 5e3cbf63..945b0cfd 100644
--- a/test/box/alter.result
+++ b/test/box/alter.result
@@ -1244,7 +1244,7 @@ t[5] = 1
 ...
 box.space._space:replace(t)
 ---
-- error: Vinyl does not support changing space format of a non-empty space
+- error: Vinyl does not support changing format of a non-empty space
 ...
 s:drop()
 ---
@@ -1558,7 +1558,7 @@ format[2] = {name = 'field2', type = 'unsigned'}
 ...
 s:format(format)
 ---
-- error: Vinyl does not support changing space format of a non-empty space
+- error: Vinyl does not support changing format of a non-empty space
 ...
 s:drop()
 ---
diff --git a/test/vinyl/ddl.result b/test/vinyl/ddl.result
index 456977e5..4ea94d36 100644
--- a/test/vinyl/ddl.result
+++ b/test/vinyl/ddl.result
@@ -83,11 +83,11 @@ space:insert({1})
 -- fail because of wrong tuple format {1}, but need {1, ...}
 index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
 ---
-- error: Vinyl does not support adding an index to a non-empty space
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space.index.primary:alter({parts = {1, 'unsigned', 2, 'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 #box.space._index:select({space.id})
 ---
@@ -112,11 +112,11 @@ space:insert({1, 2})
 ...
 index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
 ---
-- error: Vinyl does not support adding an index to a non-empty space
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space.index.primary:alter({parts = {1, 'unsigned', 2, 'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 #box.space._index:select({space.id})
 ---
@@ -141,11 +141,11 @@ space:insert({1, 2})
 ...
 index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
 ---
-- error: Vinyl does not support adding an index to a non-empty space
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space.index.primary:alter({parts = {1, 'unsigned', 2, 'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 #box.space._index:select({space.id})
 ---
@@ -161,11 +161,11 @@ space:delete({1})
 -- must fail because vy_mems have data
 index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
 ---
-- error: Vinyl does not support adding an index to a non-empty space
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space.index.primary:alter({parts = {1, 'unsigned', 2, 'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 box.snapshot()
 ---
@@ -232,11 +232,11 @@ while space.index.primary:info().run_count ~= 2 do fiber.sleep(0.01) end
 -- must fail because vy_runs have data
 index2 = space:create_index('secondary', { parts = {2, 'unsigned'} })
 ---
-- error: Vinyl does not support adding an index to a non-empty space
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space.index.primary:alter({parts = {1, 'unsigned', 2, 'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 -- After compaction the REPLACE + DELETE + DELETE = nothing, so
 -- the space is now empty and can be altered.
@@ -291,7 +291,7 @@ space:auto_increment{3}
 ...
 box.space._index:replace{space.id, 0, 'pk', 'tree', {unique=true}, {{0, 'unsigned'}, {1, 'unsigned'}}}
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 space:select{}
 ---
@@ -774,7 +774,7 @@ format[2].is_nullable = false
 ...
 space:format(format)
 ---
-- error: Vinyl does not support changing space format of a non-empty space
+- error: Vinyl does not support changing format of a non-empty space
 ...
 space:drop()
 ---
@@ -832,7 +832,7 @@ s.index.secondary.unique
 ...
 s.index.secondary:alter{unique = true} -- error
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 s.index.secondary.unique
 ---
diff --git a/test/vinyl/gh.result b/test/vinyl/gh.result
index 9c72acc8..47695acb 100644
--- a/test/vinyl/gh.result
+++ b/test/vinyl/gh.result
@@ -144,7 +144,7 @@ s:insert{5, 5}
 ...
 s.index.primary:alter({parts={2,'unsigned'}})
 ---
-- error: Vinyl does not support changing the definition of a non-empty index
+- error: Vinyl does not support building an index for a non-empty space
 ...
 s:drop()
 ---
-- 
2.11.0

Re: [PATCH 10/12] vinyl: remove superfluous ddl checks

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> There's no need to iterate over all indexes in vinyl_space_prepare_alter
> and check if any of them is going to be rebuilt - alter_space_do already
> does that for us. All we need to do is raise an error if the build_index
> callback is invoked for a non-empty space after recovery is complete.
> The tuple format check is pointless too, as it is already done by
> CheckSpaceFormat AlterSpaceOp. Remove them.

Nice. OK to push.

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 11/12] vinyl: force index rebuild if indexed field type is narrowed

[Vladimir Davydov]

If the type of an indexed field is narrowed (e.g. is_nullable flag is
cleared or the type is changed from 'integer' to 'unsigned'), but the
index structure remains the same (same fields, indexed in the same
order), we won't rebuild the index, instead we will check that all
tuples stored in the space conform to the new format (CheckSpaceFormat)
and if so modify the index in-place (ModifyIndex). This is OK for memtx,
but not for Vinyl, because even if the space contains no tuples
conflicting with the new format, there may be overwritten or deleted
statements, which can't be detected by CheckSpaceFormat.

That being said, let's make index_def_change_requires_rebuild() virtual
(add it to index_vtab) and force rebuild of Vinyl indexes if the type of
any indexed field is narrowed.
---
 src/box/alter.cc        |  7 ++-----
 src/box/index.h         | 13 +++++++++++++
 src/box/index_def.c     | 21 ---------------------
 src/box/index_def.h     | 16 ----------------
 src/box/key_def.cc      | 19 -------------------
 src/box/key_def.h       | 13 -------------
 src/box/memtx_bitset.c  |  2 ++
 src/box/memtx_engine.c  | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/box/memtx_engine.h  |  8 ++++++++
 src/box/memtx_hash.c    |  2 ++
 src/box/memtx_rtree.c   | 15 +++++++++++++++
 src/box/memtx_tree.c    |  2 ++
 src/box/sysview_index.c | 11 +++++++++++
 src/box/vinyl.c         | 45 +++++++++++++++++++++++++++++++++++++++++++++
 test/vinyl/ddl.result   | 28 ++++++++++++++++++++++++++++
 test/vinyl/ddl.test.lua | 11 +++++++++++
 16 files changed, 181 insertions(+), 74 deletions(-)

diff --git a/src/box/alter.cc b/src/box/alter.cc
index d05c6483..ac00d875 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -1476,10 +1476,7 @@ alter_space_move_indexes(struct alter_space *alter, uint32_t begin,
 					old_def->key_def, alter->pk_def);
 		index_def_update_optionality(new_def, min_field_count);
 		auto guard = make_scoped_guard([=] { index_def_delete(new_def); });
-		if (key_part_check_compatibility(old_def->cmp_def->parts,
-						 old_def->cmp_def->part_count,
-						 new_def->cmp_def->parts,
-						 new_def->cmp_def->part_count))
+		if (!index_def_change_requires_rebuild(old_index, new_def))
 			(void) new ModifyIndex(alter, new_def, old_def);
 		else
 			(void) new RebuildIndex(alter, new_def, old_def);
@@ -1852,7 +1849,7 @@ on_replace_dd_index(struct trigger * /* trigger */, void *event)
 		if (index_def_cmp(index_def, old_index->def) == 0) {
 			/* Index is not changed so just move it. */
 			(void) new MoveIndex(alter, old_index->def->iid);
-		} else if (index_def_change_requires_rebuild(old_index->def,
+		} else if (index_def_change_requires_rebuild(old_index,
 							     index_def)) {
 			/*
 			 * Operation demands an index rebuild.
diff --git a/src/box/index.h b/src/box/index.h
index f19fbd16..0aa96ade 100644
--- a/src/box/index.h
+++ b/src/box/index.h
@@ -379,6 +379,12 @@ struct index_vtab {
 	 * the primary key is modified.
 	 */
 	bool (*depends_on_pk)(struct index *);
+	/**
+	 * Return true if the change of index definition
+	 * cannot be done without rebuild.
+	 */
+	bool (*def_change_requires_rebuild)(struct index *index,
+					    const struct index_def *new_def);
 
 	ssize_t (*size)(struct index *);
 	ssize_t (*bsize)(struct index *);
@@ -517,6 +523,13 @@ index_depends_on_pk(struct index *index)
 	return index->vtab->depends_on_pk(index);
 }
 
+static inline bool
+index_def_change_requires_rebuild(struct index *index,
+				  const struct index_def *new_def)
+{
+	return index->vtab->def_change_requires_rebuild(index, new_def);
+}
+
 static inline ssize_t
 index_size(struct index *index)
 {
diff --git a/src/box/index_def.c b/src/box/index_def.c
index c503c787..38f23e0f 100644
--- a/src/box/index_def.c
+++ b/src/box/index_def.c
@@ -156,27 +156,6 @@ index_def_delete(struct index_def *index_def)
 	free(index_def);
 }
 
-bool
-index_def_change_requires_rebuild(const struct index_def *old_index_def,
-				  const struct index_def *new_index_def)
-{
-	if (old_index_def->iid != new_index_def->iid ||
-	    old_index_def->type != new_index_def->type ||
-	    (!old_index_def->opts.is_unique && new_index_def->opts.is_unique) ||
-	    !key_part_check_compatibility(old_index_def->key_def->parts,
-					  old_index_def->key_def->part_count,
-					  new_index_def->key_def->parts,
-					  new_index_def->key_def->part_count)) {
-		return true;
-	}
-	if (old_index_def->type == RTREE) {
-		if (old_index_def->opts.dimension != new_index_def->opts.dimension
-		    || old_index_def->opts.distance != new_index_def->opts.distance)
-			return true;
-	}
-	return false;
-}
-
 int
 index_def_cmp(const struct index_def *key1, const struct index_def *key2)
 {
diff --git a/src/box/index_def.h b/src/box/index_def.h
index 251506a8..eac9f06c 100644
--- a/src/box/index_def.h
+++ b/src/box/index_def.h
@@ -199,22 +199,6 @@ index_def_list_add(struct rlist *index_def_list, struct index_def *index_def)
 }
 
 /**
- * True, if the index change by alter requires an index rebuild.
- *
- * Some changes, such as a new page size or bloom_fpr do not
- * take effect immediately, so do not require a rebuild.
- *
- * Others, such as index name change, do not change the data, only
- * metadata, so do not require a rebuild either.
- *
- * Finally, changing index type or number of parts always requires
- * a rebuild.
- */
-bool
-index_def_change_requires_rebuild(const struct index_def *old_index_def,
-				  const struct index_def *new_index_def);
-
-/**
  * Create a new index definition definition.
  *
  * @param key_def  key definition, must be fully built
diff --git a/src/box/key_def.cc b/src/box/key_def.cc
index 50786913..45997ae8 100644
--- a/src/box/key_def.cc
+++ b/src/box/key_def.cc
@@ -244,25 +244,6 @@ key_part_cmp(const struct key_part *parts1, uint32_t part_count1,
 	return part_count1 < part_count2 ? -1 : part_count1 > part_count2;
 }
 
-bool
-key_part_check_compatibility(const struct key_part *old_parts,
-			     uint32_t old_part_count,
-			     const struct key_part *new_parts,
-			     uint32_t new_part_count)
-{
-	if (new_part_count != old_part_count)
-		return false;
-	for (uint32_t i = 0; i < new_part_count; i++) {
-		const struct key_part *new_part = &new_parts[i];
-		const struct key_part *old_part = &old_parts[i];
-		if (old_part->fieldno != new_part->fieldno)
-			return false;
-		if (old_part->coll != new_part->coll)
-			return false;
-	}
-	return true;
-}
-
 void
 key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno,
 		 enum field_type type, bool is_nullable, struct coll *coll)
diff --git a/src/box/key_def.h b/src/box/key_def.h
index 84e3e1e5..12016a51 100644
--- a/src/box/key_def.h
+++ b/src/box/key_def.h
@@ -429,19 +429,6 @@ key_part_cmp(const struct key_part *parts1, uint32_t part_count1,
 	     const struct key_part *parts2, uint32_t part_count2);
 
 /**
- * Find out whether alteration of an index has changed it
- * substantially enough to warrant a rebuild or not. For example,
- * change of index id is not a substantial change, whereas change
- * of index type or incompatible change of key parts requires
- * a rebuild.
- */
-bool
-key_part_check_compatibility(const struct key_part *old_parts,
-			     uint32_t old_part_count,
-			     const struct key_part *new_parts,
-			     uint32_t new_part_count);
-
-/**
  * Extract key from tuple by given key definition and return
  * buffer allocated on box_txn_alloc with this key. This function
  * has O(n) complexity, where n is the number of key parts.
diff --git a/src/box/memtx_bitset.c b/src/box/memtx_bitset.c
index f67405ec..7b87560b 100644
--- a/src/box/memtx_bitset.c
+++ b/src/box/memtx_bitset.c
@@ -462,6 +462,8 @@ static const struct index_vtab memtx_bitset_index_vtab = {
 	/* .commit_drop = */ memtx_index_commit_drop,
 	/* .update_def = */ generic_index_update_def,
 	/* .depends_on_pk = */ generic_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		memtx_index_def_change_requires_rebuild,
 	/* .size = */ memtx_bitset_index_size,
 	/* .bsize = */ memtx_bitset_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index c183b2da..388cc4fe 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -1109,3 +1109,45 @@ memtx_index_commit_drop(struct index *index)
 {
 	memtx_index_prune(index);
 }
+
+bool
+memtx_index_def_change_requires_rebuild(struct index *index,
+					const struct index_def *new_def)
+{
+	struct index_def *old_def = index->def;
+
+	assert(old_def->iid == new_def->iid);
+	assert(old_def->space_id == new_def->space_id);
+
+	if (old_def->type != new_def->type)
+		return true;
+	if (!old_def->opts.is_unique && new_def->opts.is_unique)
+		return true;
+
+	const struct key_def *old_cmp_def, *new_cmp_def;
+	if (index_depends_on_pk(index)) {
+		old_cmp_def = old_def->cmp_def;
+		new_cmp_def = new_def->cmp_def;
+	} else {
+		old_cmp_def = old_def->key_def;
+		new_cmp_def = new_def->key_def;
+	}
+
+	/*
+	 * Compatibility of field types is verified by CheckSpaceFormat
+	 * so it suffices to check that the new key definition indexes
+	 * the same set of fields in the same order.
+	 */
+	if (old_cmp_def->part_count != new_cmp_def->part_count)
+		return true;
+
+	for (uint32_t i = 0; i < new_cmp_def->part_count; i++) {
+		const struct key_part *old_part = &old_cmp_def->parts[i];
+		const struct key_part *new_part = &new_cmp_def->parts[i];
+		if (old_part->fieldno != new_part->fieldno)
+			return true;
+		if (old_part->coll != new_part->coll)
+			return true;
+	}
+	return false;
+}
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index 3f8e7db7..c1f2652c 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -161,6 +161,14 @@ memtx_index_abort_create(struct index *index);
 void
 memtx_index_commit_drop(struct index *index);
 
+/**
+ * Generic implementation of index_vtab::def_change_requires_rebuild,
+ * common for all kinds of memtx indexes.
+ */
+bool
+memtx_index_def_change_requires_rebuild(struct index *index,
+					const struct index_def *new_def);
+
 #if defined(__cplusplus)
 } /* extern "C" */
 
diff --git a/src/box/memtx_hash.c b/src/box/memtx_hash.c
index 38027a3b..24138f42 100644
--- a/src/box/memtx_hash.c
+++ b/src/box/memtx_hash.c
@@ -386,6 +386,8 @@ static const struct index_vtab memtx_hash_index_vtab = {
 	/* .commit_drop = */ memtx_index_commit_drop,
 	/* .update_def = */ memtx_hash_index_update_def,
 	/* .depends_on_pk = */ generic_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		memtx_index_def_change_requires_rebuild,
 	/* .size = */ memtx_hash_index_size,
 	/* .bsize = */ memtx_hash_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/src/box/memtx_rtree.c b/src/box/memtx_rtree.c
index d0dceaea..653343a7 100644
--- a/src/box/memtx_rtree.c
+++ b/src/box/memtx_rtree.c
@@ -154,6 +154,19 @@ memtx_rtree_index_destroy(struct index *base)
 	free(index);
 }
 
+static bool
+memtx_rtree_index_def_change_requires_rebuild(struct index *index,
+					      const struct index_def *new_def)
+{
+	if (memtx_index_def_change_requires_rebuild(index, new_def))
+		return true;
+	if (index->def->opts.distance != new_def->opts.distance ||
+	    index->def->opts.dimension != new_def->opts.dimension)
+		return true;
+	return false;
+
+}
+
 static ssize_t
 memtx_rtree_index_size(struct index *base)
 {
@@ -293,6 +306,8 @@ static const struct index_vtab memtx_rtree_index_vtab = {
 	/* .commit_drop = */ memtx_index_commit_drop,
 	/* .update_def = */ generic_index_update_def,
 	/* .depends_on_pk = */ generic_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		memtx_rtree_index_def_change_requires_rebuild,
 	/* .size = */ memtx_rtree_index_size,
 	/* .bsize = */ memtx_rtree_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/src/box/memtx_tree.c b/src/box/memtx_tree.c
index 22177f57..073006b4 100644
--- a/src/box/memtx_tree.c
+++ b/src/box/memtx_tree.c
@@ -595,6 +595,8 @@ static const struct index_vtab memtx_tree_index_vtab = {
 	/* .commit_drop = */ memtx_index_commit_drop,
 	/* .update_def = */ memtx_tree_index_update_def,
 	/* .depends_on_pk = */ memtx_tree_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		memtx_index_def_change_requires_rebuild,
 	/* .size = */ memtx_tree_index_size,
 	/* .bsize = */ memtx_tree_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/src/box/sysview_index.c b/src/box/sysview_index.c
index 72961401..38ef8108 100644
--- a/src/box/sysview_index.c
+++ b/src/box/sysview_index.c
@@ -90,6 +90,15 @@ sysview_index_bsize(struct index *index)
 	return 0;
 }
 
+static bool
+sysview_index_def_change_requires_rebuild(struct index *index,
+					  const struct index_def *new_def)
+{
+	(void)index;
+	(void)new_def;
+	return true;
+}
+
 static struct iterator *
 sysview_index_create_iterator(struct index *base, enum iterator_type type,
 			      const char *key, uint32_t part_count)
@@ -166,6 +175,8 @@ static const struct index_vtab sysview_index_vtab = {
 	/* .commit_drop = */ generic_index_commit_drop,
 	/* .update_def = */ generic_index_update_def,
 	/* .depends_on_pk = */ generic_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		sysview_index_def_change_requires_rebuild,
 	/* .size = */ generic_index_size,
 	/* .bsize = */ sysview_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 47804ecc..c99b518e 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -966,6 +966,49 @@ vinyl_index_depends_on_pk(struct index *index)
 	return true;
 }
 
+static bool
+vinyl_index_def_change_requires_rebuild(struct index *index,
+					const struct index_def *new_def)
+{
+	struct index_def *old_def = index->def;
+
+	assert(old_def->iid == new_def->iid);
+	assert(old_def->space_id == new_def->space_id);
+	assert(old_def->type == TREE && new_def->type == TREE);
+
+	if (!old_def->opts.is_unique && new_def->opts.is_unique)
+		return true;
+
+	assert(index_depends_on_pk(index));
+	const struct key_def *old_cmp_def = old_def->cmp_def;
+	const struct key_def *new_cmp_def = new_def->cmp_def;
+
+	/*
+	 * It is not enough to check only fieldno in case of Vinyl,
+	 * because the index may store some overwritten or deleted
+	 * statements conforming to the old format. CheckSpaceFormat
+	 * won't reveal such statements, but we may still need to
+	 * compare them to statements inserted after ALTER hence
+	 * we can't narrow field types without index rebuild.
+	 */
+	if (old_cmp_def->part_count != new_cmp_def->part_count)
+		return true;
+
+	for (uint32_t i = 0; i < new_cmp_def->part_count; i++) {
+		const struct key_part *old_part = &old_cmp_def->parts[i];
+		const struct key_part *new_part = &new_cmp_def->parts[i];
+		if (old_part->fieldno != new_part->fieldno)
+			return true;
+		if (old_part->coll != new_part->coll)
+			return true;
+		if (old_part->is_nullable && !new_part->is_nullable)
+			return true;
+		if (!field_type1_contains_type2(new_part->type, old_part->type))
+			return true;
+	}
+	return false;
+}
+
 static void
 vinyl_init_system_space(struct space *space)
 {
@@ -3906,6 +3949,8 @@ static const struct index_vtab vinyl_index_vtab = {
 	/* .commit_drop = */ vinyl_index_commit_drop,
 	/* .update_def = */ generic_index_update_def,
 	/* .depends_on_pk = */ vinyl_index_depends_on_pk,
+	/* .def_change_requires_rebuild = */
+		vinyl_index_def_change_requires_rebuild,
 	/* .size = */ vinyl_index_size,
 	/* .bsize = */ vinyl_index_bsize,
 	/* .min = */ generic_index_min,
diff --git a/test/vinyl/ddl.result b/test/vinyl/ddl.result
index 4ea94d36..5142f0f2 100644
--- a/test/vinyl/ddl.result
+++ b/test/vinyl/ddl.result
@@ -850,3 +850,31 @@ s.index.secondary:select(10)
 s:drop()
 ---
 ...
+-- Narrowing indexed field type entails index rebuild.
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('i1')
+---
+...
+_ = s:create_index('i2', {parts = {2, 'integer'}})
+---
+...
+_ = s:create_index('i3', {parts = {{3, 'string', is_nullable = true}}})
+---
+...
+_ = s:replace{1, 1, 'test'}
+---
+...
+-- Should fail with 'Vinyl does not support building an index for a non-empty space'.
+s.index.i2:alter{parts = {2, 'unsigned'}}
+---
+- error: Vinyl does not support building an index for a non-empty space
+...
+s.index.i3:alter{parts = {{3, 'string', is_nullable = false}}}
+---
+- error: Vinyl does not support building an index for a non-empty space
+...
+s:drop()
+---
+...
diff --git a/test/vinyl/ddl.test.lua b/test/vinyl/ddl.test.lua
index 8ce8b920..c4bd36bb 100644
--- a/test/vinyl/ddl.test.lua
+++ b/test/vinyl/ddl.test.lua
@@ -304,3 +304,14 @@ s.index.secondary.unique
 s:insert{2, 10}
 s.index.secondary:select(10)
 s:drop()
+
+-- Narrowing indexed field type entails index rebuild.
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('i1')
+_ = s:create_index('i2', {parts = {2, 'integer'}})
+_ = s:create_index('i3', {parts = {{3, 'string', is_nullable = true}}})
+_ = s:replace{1, 1, 'test'}
+-- Should fail with 'Vinyl does not support building an index for a non-empty space'.
+s.index.i2:alter{parts = {2, 'unsigned'}}
+s.index.i3:alter{parts = {{3, 'string', is_nullable = false}}}
+s:drop()
-- 
2.11.0

Re: [PATCH 11/12] vinyl: force index rebuild if indexed field type is narrowed

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> If the type of an indexed field is narrowed (e.g. is_nullable flag is
> cleared or the type is changed from 'integer' to 'unsigned'), but the
> index structure remains the same (same fields, indexed in the same
> order), we won't rebuild the index, instead we will check that all
> tuples stored in the space conform to the new format (CheckSpaceFormat)
> and if so modify the index in-place (ModifyIndex). This is OK for memtx,
> but not for Vinyl, because even if the space contains no tuples
> conflicting with the new format, there may be overwritten or deleted
> statements, which can't be detected by CheckSpaceFormat.

OK to push.

> 

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

[PATCH 12/12] vinyl: allow to modify format of non-empty spaces

[Vladimir Davydov]

This patch implements space_vtab::check_space_format callback for vinyl
spaces. The callback iterates over all tuples stored in the primary
index and validates them against the new format. Since vinyl read
iterator may yield, this callback also installs an on_replace trigger
for the altered space. The trigger checks that tuples inserted during
ALTER conform to the new format and aborts ALTER if they do not.

To test the feature, this patch enables memtx space format tests for
vinyl spaces by moving them from box/alter to engine/ddl. It adds just
one new vinyl-specific test case to vinyl/ddl. The test case checks
that tuples inserted during ALTER are validated against the new format.
---
 src/box/vinyl.c            |  95 +++++-
 test/box/alter.result      | 810 +--------------------------------------------
 test/box/alter.test.lua    | 319 ------------------
 test/engine/ddl.result     | 787 ++++++++++++++++++++++++++++++++++++++++++-
 test/engine/ddl.test.lua   | 311 ++++++++++++++++-
 test/vinyl/ddl.result      |  83 -----
 test/vinyl/ddl.test.lua    |  26 --
 test/vinyl/errinj.result   |  75 +++++
 test/vinyl/errinj.test.lua |  35 ++
 9 files changed, 1288 insertions(+), 1253 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index c99b518e..c2769e6d 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1028,21 +1028,98 @@ vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
 	return 0;
 }
 
+/** Argument passed to vy_check_format_on_replace(). */
+struct vy_check_format_ctx {
+	/** Format to check new tuples against. */
+	struct tuple_format *format;
+	/** Set if a new tuple doesn't conform to the format. */
+	bool is_failed;
+	/** Container for storing errors. */
+	struct diag diag;
+};
+
+/**
+ * This is an on_replace trigger callback that checks inserted
+ * tuples against a new format.
+ */
+static void
+vy_check_format_on_replace(struct trigger *trigger, void *event)
+{
+	struct txn *txn = event;
+	struct txn_stmt *stmt = txn_current_stmt(txn);
+	struct vy_check_format_ctx *ctx = trigger->data;
+
+	if (stmt->new_tuple == NULL)
+		return; /* DELETE, nothing to do */
+
+	if (ctx->is_failed)
+		return; /* already failed, nothing to do */
+
+	if (tuple_validate(ctx->format, stmt->new_tuple) != 0) {
+		ctx->is_failed = true;
+		diag_move(diag_get(), &ctx->diag);
+	}
+}
+
 static int
 vinyl_space_check_format(struct space *space, struct tuple_format *format)
 {
-	(void)format;
 	struct vy_env *env = vy_env(space->engine);
-	if (space->index_count == 0)
+
+	/*
+	 * If this is local recovery, the space was checked before
+	 * restart so there's nothing we need to do.
+	 */
+	if (env->status == VINYL_INITIAL_RECOVERY_LOCAL ||
+	    env->status == VINYL_FINAL_RECOVERY_LOCAL)
 		return 0;
+
+	if (space->index_count == 0)
+		return 0; /* space is empty, nothing to do */
+
+	/*
+	 * Iterate over all tuples stored in the given space and
+	 * check each of them for conformity to the new format.
+	 * Since read iterator may yield, we install an on_replace
+	 * trigger to check tuples inserted after we started the
+	 * iteration.
+	 */
 	struct vy_lsm *pk = vy_lsm(space->index[0]);
-	if (env->status != VINYL_ONLINE)
-		return 0;
-	if (pk->stat.disk.count.rows == 0 && pk->stat.memory.count.rows == 0)
-		return 0;
-	diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
-		 "changing format of a non-empty space");
-	return -1;
+
+	struct tuple *key = vy_stmt_new_select(pk->env->key_format, NULL, 0);
+	if (key == NULL)
+		return -1;
+
+	struct trigger on_replace;
+	struct vy_check_format_ctx ctx;
+	ctx.format = format;
+	ctx.is_failed = false;
+	diag_create(&ctx.diag);
+	trigger_create(&on_replace, vy_check_format_on_replace, &ctx, NULL);
+	trigger_add(&space->on_replace, &on_replace);
+
+	struct vy_read_iterator itr;
+	vy_read_iterator_open(&itr, pk, NULL, ITER_ALL, key,
+			      &env->xm->p_global_read_view);
+	int rc;
+	struct tuple *tuple;
+	while ((rc = vy_read_iterator_next(&itr, &tuple)) == 0) {
+		if (tuple == NULL)
+			break;
+		if (ctx.is_failed) {
+			diag_move(&ctx.diag, diag_get());
+			rc = -1;
+			break;
+		}
+		rc = tuple_validate(format, tuple);
+		if (rc != 0)
+			break;
+	}
+	vy_read_iterator_close(&itr);
+	diag_destroy(&ctx.diag);
+	trigger_clear(&on_replace);
+	tuple_unref(key);
+	return rc;
 }
 
 static void
diff --git a/test/box/alter.result b/test/box/alter.result
index 945b0cfd..a78dd3e9 100644
--- a/test/box/alter.result
+++ b/test/box/alter.result
@@ -817,753 +817,6 @@ ts:drop()
 ---
 ...
 --
--- gh-2652: validate space format.
---
-s = box.schema.space.create('test', { format = "format" })
----
-- error: Illegal parameters, options parameter 'format' should be of type table
-...
-format = { { name = 100 } }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-- error: 'Illegal parameters, format[1]: name (string) is expected'
-...
-long = string.rep('a', box.schema.NAME_MAX + 1)
----
-...
-format = { { name = long } }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-- error: 'Failed to create space ''test'': field 1 name is too long'
-...
-format = { { name = 'id', type = '100' } }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-- error: 'Failed to create space ''test'': field 1 has unknown field type'
-...
-format = { utils.setmap({}) }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-- error: 'Illegal parameters, format[1]: name (string) is expected'
-...
--- Ensure the format is updated after index drop.
-format = { { name = 'id', type = 'unsigned' } }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-...
-pk = s:create_index('pk')
----
-...
-sk = s:create_index('sk', { parts = { 2, 'string' } })
----
-...
-s:replace{1, 1}
----
-- error: 'Tuple field 2 type does not match one required by operation: expected string'
-...
-sk:drop()
----
-...
-s:replace{1, 1}
----
-- [1, 1]
-...
-s:drop()
----
-...
--- Check index parts conflicting with space format.
-format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
----
-...
-s = box.schema.space.create('test', { format = format })
----
-...
-pk = s:create_index('pk')
----
-...
-sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
----
-- error: Field 'field2' has type 'string' in space format, but type 'unsigned' in
-    index definition
-...
--- Check space format conflicting with index parts.
-sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
----
-...
-format[2].type = 'unsigned'
----
-...
-s:format(format)
----
-- error: Field 'field2' has type 'unsigned' in space format, but type 'string' in
-    index definition
-...
-s:format()
----
-- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'string'}, {
-    'name': 'field3', 'type': 'scalar'}]
-...
-s.index.sk3.parts
----
-- - type: string
-    is_nullable: false
-    fieldno: 2
-...
--- Space format can be updated, if conflicted index is deleted.
-sk3:drop()
----
-...
-s:format(format)
----
-...
-s:format()
----
-- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'unsigned'},
-  {'name': 'field3', 'type': 'scalar'}]
-...
--- Check deprecated field types.
-format[2].type = 'num'
----
-...
-format[3].type = 'str'
----
-...
-format[4] = { name = 'field4', type = '*' }
----
-...
-format
----
-- - name: field1
-    type: unsigned
-  - name: field2
-    type: num
-  - name: field3
-    type: str
-  - name: field4
-    type: '*'
-...
-s:format(format)
----
-...
-s:format()
----
-- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'num'}, {'name': 'field3',
-    'type': 'str'}, {'name': 'field4', 'type': '*'}]
-...
-s:replace{1, 2, '3', {4, 4, 4}}
----
-- [1, 2, '3', [4, 4, 4]]
-...
--- Check not indexed fields checking.
-s:truncate()
----
-...
-format[2] = {name='field2', type='string'}
----
-...
-format[3] = {name='field3', type='array'}
----
-...
-format[4] = {name='field4', type='number'}
----
-...
-format[5] = {name='field5', type='integer'}
----
-...
-format[6] = {name='field6', type='scalar'}
----
-...
-format[7] = {name='field7', type='map'}
----
-...
-format[8] = {name='field8', type='any'}
----
-...
-format[9] = {name='field9'}
----
-...
-s:format(format)
----
-...
--- Check incorrect field types.
-format[9] = {name='err', type='any'}
----
-...
-s:format(format)
----
-...
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
----
-- [1, '2', [3, 3], 4.4, -5, true, {'value': 7}, 8, 9]
-...
-s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
----
-- error: 'Tuple field 2 type does not match one required by operation: expected string'
-...
-s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
----
-- error: 'Tuple field 3 type does not match one required by operation: expected array'
-...
-s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
----
-- error: 'Tuple field 4 type does not match one required by operation: expected number'
-...
-s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
----
-- error: 'Tuple field 5 type does not match one required by operation: expected integer'
-...
-s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
----
-- error: 'Tuple field 6 type does not match one required by operation: expected scalar'
-...
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
----
-- error: 'Tuple field 7 type does not match one required by operation: expected map'
-...
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
----
-- error: Tuple field count 7 is less than required by space format or defined indexes
-    (expected at least 9)
-...
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
----
-- error: Tuple field count 8 is less than required by space format or defined indexes
-    (expected at least 9)
-...
-s:truncate()
----
-...
---
--- gh-1014: field names.
---
-format = {}
----
-...
-format[1] = {name = 'field1', type = 'unsigned'}
----
-...
-format[2] = {name = 'field2'}
----
-...
-format[3] = {name = 'field1'}
----
-...
-s:format(format)
----
-- error: Space field 'field1' is duplicate
-...
-s:drop()
----
-...
--- https://github.com/tarantool/tarantool/issues/2815
--- Extend space format definition syntax
-format = {{name='key',type='unsigned'}, {name='value',type='string'}}
----
-...
-s = box.schema.space.create('test', { format = format })
----
-...
-s:format()
----
-- [{'name': 'key', 'type': 'unsigned'}, {'name': 'value', 'type': 'string'}]
-...
-s:format({'id', 'name'})
----
-...
-s:format()
----
-- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'any'}]
-...
-s:format({'id', {'name1'}})
----
-...
-s:format()
----
-- [{'name': 'id', 'type': 'any'}, {'name': 'name1', 'type': 'any'}]
-...
-s:format({'id', {'name2', 'string'}})
----
-...
-s:format()
----
-- [{'name': 'id', 'type': 'any'}, {'name': 'name2', 'type': 'string'}]
-...
-s:format({'id', {'name', type = 'string'}})
----
-...
-s:format()
----
-- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'string'}]
-...
-s:drop()
----
-...
-format = {'key', {'value',type='string'}}
----
-...
-s = box.schema.space.create('test', { format = format })
----
-...
-s:format()
----
-- [{'name': 'key', 'type': 'any'}, {'name': 'value', 'type': 'string'}]
-...
-s:drop()
----
-...
-s = box.schema.space.create('test')
----
-...
-s:create_index('test', {parts = {'test'}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test', {parts = {{'test'}}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test', {parts = {{field = 'test'}}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test', {parts = {1}}).parts
----
-- - type: scalar
-    is_nullable: false
-    fieldno: 1
-...
-s:drop()
----
-...
-s = box.schema.space.create('test')
----
-...
-s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
----
-...
-s:create_index('test', {parts = {'test'}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test', {parts = {{'test'}}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test', {parts = {{field = 'test'}}})
----
-- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
-...
-s:create_index('test1', {parts = {'test1'}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 1
-...
-s:create_index('test2', {parts = {'test2'}}).parts
----
-- error: 'Can''t create or modify index ''test2'' in space ''test'': field type ''any''
-    is not supported'
-...
-s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 1
-...
-s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 2
-...
-s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 2
-...
-s:create_index('test6', {parts = {1, 3}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 1
-  - type: integer
-    is_nullable: false
-    fieldno: 3
-...
-s:create_index('test7', {parts = {'test1', 4}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 1
-  - type: scalar
-    is_nullable: false
-    fieldno: 4
-...
-s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
----
-- - type: integer
-    is_nullable: false
-    fieldno: 1
-  - type: scalar
-    is_nullable: false
-    fieldno: 4
-...
-s:drop()
----
-...
---
--- gh-2800: space formats checking is broken.
---
--- Ensure that vinyl correctly process field count change.
-s = box.schema.space.create('test', {engine = 'vinyl', field_count = 2})
----
-...
-pk = s:create_index('pk')
----
-...
-s:replace{1, 2}
----
-- [1, 2]
-...
-t = box.space._space:select{s.id}[1]:totable()
----
-...
-t[5] = 1
----
-...
-box.space._space:replace(t)
----
-- error: Vinyl does not support changing format of a non-empty space
-...
-s:drop()
----
-...
--- Check field type changes.
-format = {}
----
-...
-format[1] = {name = 'field1', type = 'unsigned'}
----
-...
-format[2] = {name = 'field2', type = 'any'}
----
-...
-format[3] = {name = 'field3', type = 'unsigned'}
----
-...
-format[4] = {name = 'field4', type = 'string'}
----
-...
-format[5] = {name = 'field5', type = 'number'}
----
-...
-format[6] = {name = 'field6', type = 'integer'}
----
-...
-format[7] = {name = 'field7', type = 'boolean'}
----
-...
-format[8] = {name = 'field8', type = 'scalar'}
----
-...
-format[9] = {name = 'field9', type = 'array'}
----
-...
-format[10] = {name = 'field10', type = 'map'}
----
-...
-s = box.schema.space.create('test', {format = format})
----
-...
-pk = s:create_index('pk')
----
-...
-t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
----
-...
-test_run:cmd("setopt delimiter ';'")
----
-- true
-...
-function fail_format_change(fieldno, new_type)
-    local old_type = format[fieldno].type
-    format[fieldno].type = new_type
-    local ok, msg = pcall(s.format, s, format)
-    format[fieldno].type = old_type
-    return msg
-end;
----
-...
-function ok_format_change(fieldno, new_type)
-    local old_type = format[fieldno].type
-    format[fieldno].type = new_type
-    s:format(format)
-    s:delete{1}
-    format[fieldno].type = old_type
-    s:format(format)
-    s:replace(t)
-end;
----
-...
-test_run:cmd("setopt delimiter ''");
----
-- true
-...
--- any --X--> unsigned
-fail_format_change(2, 'unsigned')
----
-- 'Tuple field 2 type does not match one required by operation: expected unsigned'
-...
--- unsigned -----> any
-ok_format_change(3, 'any')
----
-...
--- unsigned --X--> string
-fail_format_change(3, 'string')
----
-- 'Tuple field 3 type does not match one required by operation: expected string'
-...
--- unsigned -----> number
-ok_format_change(3, 'number')
----
-...
--- unsigned -----> integer
-ok_format_change(3, 'integer')
----
-...
--- unsigned -----> scalar
-ok_format_change(3, 'scalar')
----
-...
--- unsigned --X--> map
-fail_format_change(3, 'map')
----
-- 'Tuple field 3 type does not match one required by operation: expected map'
-...
--- string -----> any
-ok_format_change(4, 'any')
----
-...
--- string -----> scalar
-ok_format_change(4, 'scalar')
----
-...
--- string --X--> boolean
-fail_format_change(4, 'boolean')
----
-- 'Tuple field 4 type does not match one required by operation: expected boolean'
-...
--- number -----> any
-ok_format_change(5, 'any')
----
-...
--- number -----> scalar
-ok_format_change(5, 'scalar')
----
-...
--- number --X--> integer
-fail_format_change(5, 'integer')
----
-- 'Tuple field 5 type does not match one required by operation: expected integer'
-...
--- integer -----> any
-ok_format_change(6, 'any')
----
-...
--- integer -----> number
-ok_format_change(6, 'number')
----
-...
--- integer -----> scalar
-ok_format_change(6, 'scalar')
----
-...
--- integer --X--> unsigned
-fail_format_change(6, 'unsigned')
----
-- 'Tuple field 6 type does not match one required by operation: expected unsigned'
-...
--- boolean -----> any
-ok_format_change(7, 'any')
----
-...
--- boolean -----> scalar
-ok_format_change(7, 'scalar')
----
-...
--- boolean --X--> string
-fail_format_change(7, 'string')
----
-- 'Tuple field 7 type does not match one required by operation: expected string'
-...
--- scalar -----> any
-ok_format_change(8, 'any')
----
-...
--- scalar --X--> unsigned
-fail_format_change(8, 'unsigned')
----
-- 'Tuple field 8 type does not match one required by operation: expected unsigned'
-...
--- array -----> any
-ok_format_change(9, 'any')
----
-...
--- array --X--> scalar
-fail_format_change(9, 'scalar')
----
-- 'Tuple field 9 type does not match one required by operation: expected scalar'
-...
--- map -----> any
-ok_format_change(10, 'any')
----
-...
--- map --X--> scalar
-fail_format_change(10, 'scalar')
----
-- 'Tuple field 10 type does not match one required by operation: expected scalar'
-...
-s:drop()
----
-...
--- Check new fields adding.
-format = {}
----
-...
-s = box.schema.space.create('test')
----
-...
-format[1] = {name = 'field1', type = 'unsigned'}
----
-...
-s:format(format) -- Ok, no indexes.
----
-...
-pk = s:create_index('pk')
----
-...
-format[2] = {name = 'field2', type = 'unsigned'}
----
-...
-s:format(format) -- Ok, empty space.
----
-...
-s:replace{1, 1}
----
-- [1, 1]
-...
-format[2] = nil
----
-...
-s:format(format) -- Ok, can delete fields with no checks.
----
-...
-s:delete{1}
----
-- [1, 1]
-...
-sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
----
-...
-sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
----
-...
-sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
----
-...
-s:replace{1, 1, 1, 1, 1}
----
-- [1, 1, 1, 1, 1]
-...
-format[2] = {name = 'field2', type = 'unsigned'}
----
-...
-format[3] = {name = 'field3', type = 'unsigned'}
----
-...
-format[4] = {name = 'field4', type = 'any'}
----
-...
-format[5] = {name = 'field5', type = 'unsigned'}
----
-...
--- Ok, all new fields are indexed or have type ANY, and new
--- field_count <= old field_count.
-s:format(format)
----
-...
-s:replace{1, 1, 1, 1, 1, 1}
----
-- [1, 1, 1, 1, 1, 1]
-...
-format[6] = {name = 'field6', type = 'unsigned'}
----
-...
--- Ok, but check existing tuples for a new field[6].
-s:format(format)
----
-...
--- Fail, not enough fields.
-s:replace{2, 2, 2, 2, 2}
----
-- error: Tuple field count 5 is less than required by space format or defined indexes
-    (expected at least 6)
-...
-s:replace{2, 2, 2, 2, 2, 2, 2}
----
-- [2, 2, 2, 2, 2, 2, 2]
-...
-format[7] = {name = 'field7', type = 'unsigned'}
----
-...
--- Fail, the tuple {1, ... 1} is invalid for a new format.
-s:format(format)
----
-- error: Tuple field count 6 is less than required by space format or defined indexes
-    (expected at least 7)
-...
-s:drop()
----
-...
--- Vinyl does not support adding fields to a not empty space.
-s = box.schema.space.create('test', {engine = 'vinyl'})
----
-...
-pk = s:create_index('pk')
----
-...
-s:replace{1,1}
----
-- [1, 1]
-...
-format = {}
----
-...
-format[1] = {name = 'field1', type = 'unsigned'}
----
-...
-format[2] = {name = 'field2', type = 'unsigned'}
----
-...
-s:format(format)
----
-- error: Vinyl does not support changing format of a non-empty space
-...
-s:drop()
----
-...
---
 -- gh-1557: NULL in indexes.
 --
 NULL = require('msgpack').NULL
@@ -1607,7 +860,7 @@ s:create_index('primary', { parts = {'field1'} })
     is_nullable: false
     fieldno: 1
   id: 0
-  space_id: 747
+  space_id: 733
   name: primary
   type: TREE
 ...
@@ -1684,7 +937,7 @@ s:create_index('secondary', { parts = {{2, 'string', is_nullable = true}} })
     is_nullable: true
     fieldno: 2
   id: 1
-  space_id: 747
+  space_id: 733
   name: secondary
   type: TREE
 ...
@@ -1804,65 +1057,6 @@ s:drop()
 ---
 ...
 --
--- Allow to restrict space format, if corresponding restrictions
--- already are defined in indexes.
---
-test_run:cmd("setopt delimiter ';'")
----
-- true
-...
-function check_format_restriction(engine, name)
-    local s = box.schema.create_space(name, {engine = engine})
-    local pk = s:create_index('pk')
-    local format = {}
-    format[1] = {name = 'field1'}
-    s:replace{1}
-    s:replace{100}
-    s:replace{0}
-    s:format(format)
-    s:format()
-    format[1].type = 'unsigned'
-    s:format(format)
-end;
----
-...
-test_run:cmd("setopt delimiter ''");
----
-- true
-...
-check_format_restriction('memtx', 'test1')
----
-...
-check_format_restriction('vinyl', 'test2')
----
-...
-box.space.test1:format()
----
-- [{'name': 'field1', 'type': 'unsigned'}]
-...
-box.space.test1:select{}
----
-- - [0]
-  - [1]
-  - [100]
-...
-box.space.test2:format()
----
-- [{'name': 'field1', 'type': 'unsigned'}]
-...
-box.space.test2:select{}
----
-- - [0]
-  - [1]
-  - [100]
-...
-box.space.test1:drop()
----
-...
-box.space.test2:drop()
----
-...
---
 -- Allow to change is_nullable in index definition on non-empty
 -- space.
 --
diff --git a/test/box/alter.test.lua b/test/box/alter.test.lua
index f6b2eb49..ab713584 100644
--- a/test/box/alter.test.lua
+++ b/test/box/alter.test.lua
@@ -316,297 +316,6 @@ n
 ts:drop()
 
 --
--- gh-2652: validate space format.
---
-s = box.schema.space.create('test', { format = "format" })
-format = { { name = 100 } }
-s = box.schema.space.create('test', { format = format })
-long = string.rep('a', box.schema.NAME_MAX + 1)
-format = { { name = long } }
-s = box.schema.space.create('test', { format = format })
-format = { { name = 'id', type = '100' } }
-s = box.schema.space.create('test', { format = format })
-format = { utils.setmap({}) }
-s = box.schema.space.create('test', { format = format })
-
--- Ensure the format is updated after index drop.
-format = { { name = 'id', type = 'unsigned' } }
-s = box.schema.space.create('test', { format = format })
-pk = s:create_index('pk')
-sk = s:create_index('sk', { parts = { 2, 'string' } })
-s:replace{1, 1}
-sk:drop()
-s:replace{1, 1}
-s:drop()
-
--- Check index parts conflicting with space format.
-format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
-s = box.schema.space.create('test', { format = format })
-pk = s:create_index('pk')
-sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
-
--- Check space format conflicting with index parts.
-sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
-format[2].type = 'unsigned'
-s:format(format)
-s:format()
-s.index.sk3.parts
-
--- Space format can be updated, if conflicted index is deleted.
-sk3:drop()
-s:format(format)
-s:format()
-
--- Check deprecated field types.
-format[2].type = 'num'
-format[3].type = 'str'
-format[4] = { name = 'field4', type = '*' }
-format
-s:format(format)
-s:format()
-s:replace{1, 2, '3', {4, 4, 4}}
-
--- Check not indexed fields checking.
-s:truncate()
-format[2] = {name='field2', type='string'}
-format[3] = {name='field3', type='array'}
-format[4] = {name='field4', type='number'}
-format[5] = {name='field5', type='integer'}
-format[6] = {name='field6', type='scalar'}
-format[7] = {name='field7', type='map'}
-format[8] = {name='field8', type='any'}
-format[9] = {name='field9'}
-s:format(format)
-
--- Check incorrect field types.
-format[9] = {name='err', type='any'}
-s:format(format)
-
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
-s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
-s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
-s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
-s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
-s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
-s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
-s:truncate()
-
---
--- gh-1014: field names.
---
-format = {}
-format[1] = {name = 'field1', type = 'unsigned'}
-format[2] = {name = 'field2'}
-format[3] = {name = 'field1'}
-s:format(format)
-
-s:drop()
-
--- https://github.com/tarantool/tarantool/issues/2815
--- Extend space format definition syntax
-format = {{name='key',type='unsigned'}, {name='value',type='string'}}
-s = box.schema.space.create('test', { format = format })
-s:format()
-s:format({'id', 'name'})
-s:format()
-s:format({'id', {'name1'}})
-s:format()
-s:format({'id', {'name2', 'string'}})
-s:format()
-s:format({'id', {'name', type = 'string'}})
-s:format()
-s:drop()
-
-format = {'key', {'value',type='string'}}
-s = box.schema.space.create('test', { format = format })
-s:format()
-s:drop()
-
-s = box.schema.space.create('test')
-s:create_index('test', {parts = {'test'}})
-s:create_index('test', {parts = {{'test'}}})
-s:create_index('test', {parts = {{field = 'test'}}})
-s:create_index('test', {parts = {1}}).parts
-s:drop()
-
-s = box.schema.space.create('test')
-s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
-s:create_index('test', {parts = {'test'}})
-s:create_index('test', {parts = {{'test'}}})
-s:create_index('test', {parts = {{field = 'test'}}})
-s:create_index('test1', {parts = {'test1'}}).parts
-s:create_index('test2', {parts = {'test2'}}).parts
-s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
-s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
-s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
-s:create_index('test6', {parts = {1, 3}}).parts
-s:create_index('test7', {parts = {'test1', 4}}).parts
-s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
-s:drop()
-
---
--- gh-2800: space formats checking is broken.
---
-
--- Ensure that vinyl correctly process field count change.
-s = box.schema.space.create('test', {engine = 'vinyl', field_count = 2})
-pk = s:create_index('pk')
-s:replace{1, 2}
-t = box.space._space:select{s.id}[1]:totable()
-t[5] = 1
-box.space._space:replace(t)
-s:drop()
-
--- Check field type changes.
-format = {}
-format[1] = {name = 'field1', type = 'unsigned'}
-format[2] = {name = 'field2', type = 'any'}
-format[3] = {name = 'field3', type = 'unsigned'}
-format[4] = {name = 'field4', type = 'string'}
-format[5] = {name = 'field5', type = 'number'}
-format[6] = {name = 'field6', type = 'integer'}
-format[7] = {name = 'field7', type = 'boolean'}
-format[8] = {name = 'field8', type = 'scalar'}
-format[9] = {name = 'field9', type = 'array'}
-format[10] = {name = 'field10', type = 'map'}
-s = box.schema.space.create('test', {format = format})
-pk = s:create_index('pk')
-t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
-
-test_run:cmd("setopt delimiter ';'")
-function fail_format_change(fieldno, new_type)
-    local old_type = format[fieldno].type
-    format[fieldno].type = new_type
-    local ok, msg = pcall(s.format, s, format)
-    format[fieldno].type = old_type
-    return msg
-end;
-
-function ok_format_change(fieldno, new_type)
-    local old_type = format[fieldno].type
-    format[fieldno].type = new_type
-    s:format(format)
-    s:delete{1}
-    format[fieldno].type = old_type
-    s:format(format)
-    s:replace(t)
-end;
-test_run:cmd("setopt delimiter ''");
-
--- any --X--> unsigned
-fail_format_change(2, 'unsigned')
-
--- unsigned -----> any
-ok_format_change(3, 'any')
--- unsigned --X--> string
-fail_format_change(3, 'string')
--- unsigned -----> number
-ok_format_change(3, 'number')
--- unsigned -----> integer
-ok_format_change(3, 'integer')
--- unsigned -----> scalar
-ok_format_change(3, 'scalar')
--- unsigned --X--> map
-fail_format_change(3, 'map')
-
--- string -----> any
-ok_format_change(4, 'any')
--- string -----> scalar
-ok_format_change(4, 'scalar')
--- string --X--> boolean
-fail_format_change(4, 'boolean')
-
--- number -----> any
-ok_format_change(5, 'any')
--- number -----> scalar
-ok_format_change(5, 'scalar')
--- number --X--> integer
-fail_format_change(5, 'integer')
-
--- integer -----> any
-ok_format_change(6, 'any')
--- integer -----> number
-ok_format_change(6, 'number')
--- integer -----> scalar
-ok_format_change(6, 'scalar')
--- integer --X--> unsigned
-fail_format_change(6, 'unsigned')
-
--- boolean -----> any
-ok_format_change(7, 'any')
--- boolean -----> scalar
-ok_format_change(7, 'scalar')
--- boolean --X--> string
-fail_format_change(7, 'string')
-
--- scalar -----> any
-ok_format_change(8, 'any')
--- scalar --X--> unsigned
-fail_format_change(8, 'unsigned')
-
--- array -----> any
-ok_format_change(9, 'any')
--- array --X--> scalar
-fail_format_change(9, 'scalar')
-
--- map -----> any
-ok_format_change(10, 'any')
--- map --X--> scalar
-fail_format_change(10, 'scalar')
-
-s:drop()
-
--- Check new fields adding.
-format = {}
-s = box.schema.space.create('test')
-format[1] = {name = 'field1', type = 'unsigned'}
-s:format(format) -- Ok, no indexes.
-pk = s:create_index('pk')
-format[2] = {name = 'field2', type = 'unsigned'}
-s:format(format) -- Ok, empty space.
-s:replace{1, 1}
-format[2] = nil
-s:format(format) -- Ok, can delete fields with no checks.
-s:delete{1}
-sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
-sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
-sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
-s:replace{1, 1, 1, 1, 1}
-format[2] = {name = 'field2', type = 'unsigned'}
-format[3] = {name = 'field3', type = 'unsigned'}
-format[4] = {name = 'field4', type = 'any'}
-format[5] = {name = 'field5', type = 'unsigned'}
--- Ok, all new fields are indexed or have type ANY, and new
--- field_count <= old field_count.
-s:format(format)
-
-s:replace{1, 1, 1, 1, 1, 1}
-format[6] = {name = 'field6', type = 'unsigned'}
--- Ok, but check existing tuples for a new field[6].
-s:format(format)
-
--- Fail, not enough fields.
-s:replace{2, 2, 2, 2, 2}
-
-s:replace{2, 2, 2, 2, 2, 2, 2}
-format[7] = {name = 'field7', type = 'unsigned'}
--- Fail, the tuple {1, ... 1} is invalid for a new format.
-s:format(format)
-s:drop()
-
--- Vinyl does not support adding fields to a not empty space.
-s = box.schema.space.create('test', {engine = 'vinyl'})
-pk = s:create_index('pk')
-s:replace{1,1}
-format = {}
-format[1] = {name = 'field1', type = 'unsigned'}
-format[2] = {name = 'field2', type = 'unsigned'}
-s:format(format)
-s:drop()
-
---
 -- gh-1557: NULL in indexes.
 --
 
@@ -696,34 +405,6 @@ s:select{}
 s:drop()
 
 --
--- Allow to restrict space format, if corresponding restrictions
--- already are defined in indexes.
---
-test_run:cmd("setopt delimiter ';'")
-function check_format_restriction(engine, name)
-    local s = box.schema.create_space(name, {engine = engine})
-    local pk = s:create_index('pk')
-    local format = {}
-    format[1] = {name = 'field1'}
-    s:replace{1}
-    s:replace{100}
-    s:replace{0}
-    s:format(format)
-    s:format()
-    format[1].type = 'unsigned'
-    s:format(format)
-end;
-test_run:cmd("setopt delimiter ''");
-check_format_restriction('memtx', 'test1')
-check_format_restriction('vinyl', 'test2')
-box.space.test1:format()
-box.space.test1:select{}
-box.space.test2:format()
-box.space.test2:select{}
-box.space.test1:drop()
-box.space.test2:drop()
-
---
 -- Allow to change is_nullable in index definition on non-empty
 -- space.
 --
diff --git a/test/engine/ddl.result b/test/engine/ddl.result
index 308aefb0..04062ac1 100644
--- a/test/engine/ddl.result
+++ b/test/engine/ddl.result
@@ -357,7 +357,7 @@ space:drop()
 format = {{'field1', 'scalar'}}
 ---
 ...
-s = box.schema.create_space('test', {format = format})
+s = box.schema.space.create('test', {engine = engine, format = format})
 ---
 ...
 pk = s:create_index('pk')
@@ -374,7 +374,7 @@ s:drop()
 format = {{'field1'}}
 ---
 ...
-s = box.schema.create_space('test', {format = format})
+s = box.schema.space.create('test', {engine = engine, format = format})
 ---
 ...
 pk = s:create_index('pk')
@@ -391,7 +391,7 @@ s:drop()
 -- gh-3229: update optionality if a space format is changed too,
 -- not only when indexes are updated.
 --
-s = box.schema.create_space('test', {engine = engine})
+s = box.schema.space.create('test', {engine = engine})
 ---
 ...
 format = {}
@@ -466,7 +466,7 @@ s:drop()
 --
 -- Modify key definition without index rebuild.
 --
-s = box.schema.create_space('test', {engine = engine})
+s = box.schema.space.create('test', {engine = engine})
 ---
 ...
 i1 = s:create_index('i1', {unique = true,  parts = {1, 'unsigned'}})
@@ -573,3 +573,782 @@ i3:select()
 s:drop()
 ---
 ...
+--
+-- gh-2652: validate space format.
+--
+s = box.schema.space.create('test', { engine = engine, format = "format" })
+---
+- error: Illegal parameters, options parameter 'format' should be of type table
+...
+format = { { name = 100 } }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+- error: 'Illegal parameters, format[1]: name (string) is expected'
+...
+long = string.rep('a', box.schema.NAME_MAX + 1)
+---
+...
+format = { { name = long } }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+- error: 'Failed to create space ''test'': field 1 name is too long'
+...
+format = { { name = 'id', type = '100' } }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+- error: 'Failed to create space ''test'': field 1 has unknown field type'
+...
+format = { setmetatable({}, { __serialize = 'map' }) }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+- error: 'Illegal parameters, format[1]: name (string) is expected'
+...
+-- Ensure the format is updated after index drop.
+format = { { name = 'id', type = 'unsigned' } }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+...
+pk = s:create_index('pk')
+---
+...
+sk = s:create_index('sk', { parts = { 2, 'string' } })
+---
+...
+s:replace{1, 1}
+---
+- error: 'Tuple field 2 type does not match one required by operation: expected string'
+...
+sk:drop()
+---
+...
+s:replace{1, 1}
+---
+- [1, 1]
+...
+s:drop()
+---
+...
+-- Check index parts conflicting with space format.
+format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+...
+pk = s:create_index('pk')
+---
+...
+sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
+---
+- error: Field 'field2' has type 'string' in space format, but type 'unsigned' in
+    index definition
+...
+-- Check space format conflicting with index parts.
+sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
+---
+...
+format[2].type = 'unsigned'
+---
+...
+s:format(format)
+---
+- error: Field 'field2' has type 'unsigned' in space format, but type 'string' in
+    index definition
+...
+s:format()
+---
+- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'string'}, {
+    'name': 'field3', 'type': 'scalar'}]
+...
+s.index.sk3.parts
+---
+- - type: string
+    is_nullable: false
+    fieldno: 2
+...
+-- Space format can be updated, if conflicted index is deleted.
+sk3:drop()
+---
+...
+s:format(format)
+---
+...
+s:format()
+---
+- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'unsigned'},
+  {'name': 'field3', 'type': 'scalar'}]
+...
+-- Check deprecated field types.
+format[2].type = 'num'
+---
+...
+format[3].type = 'str'
+---
+...
+format[4] = { name = 'field4', type = '*' }
+---
+...
+format
+---
+- - name: field1
+    type: unsigned
+  - name: field2
+    type: num
+  - name: field3
+    type: str
+  - name: field4
+    type: '*'
+...
+s:format(format)
+---
+...
+s:format()
+---
+- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'num'}, {'name': 'field3',
+    'type': 'str'}, {'name': 'field4', 'type': '*'}]
+...
+s:replace{1, 2, '3', {4, 4, 4}}
+---
+- [1, 2, '3', [4, 4, 4]]
+...
+-- Check not indexed fields checking.
+s:truncate()
+---
+...
+format[2] = {name='field2', type='string'}
+---
+...
+format[3] = {name='field3', type='array'}
+---
+...
+format[4] = {name='field4', type='number'}
+---
+...
+format[5] = {name='field5', type='integer'}
+---
+...
+format[6] = {name='field6', type='scalar'}
+---
+...
+format[7] = {name='field7', type='map'}
+---
+...
+format[8] = {name='field8', type='any'}
+---
+...
+format[9] = {name='field9'}
+---
+...
+s:format(format)
+---
+...
+-- Check incorrect field types.
+format[9] = {name='err', type='any'}
+---
+...
+s:format(format)
+---
+...
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
+---
+- [1, '2', [3, 3], 4.4, -5, true, {'value': 7}, 8, 9]
+...
+s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
+---
+- error: 'Tuple field 2 type does not match one required by operation: expected string'
+...
+s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
+---
+- error: 'Tuple field 3 type does not match one required by operation: expected array'
+...
+s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
+---
+- error: 'Tuple field 4 type does not match one required by operation: expected number'
+...
+s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
+---
+- error: 'Tuple field 5 type does not match one required by operation: expected integer'
+...
+s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
+---
+- error: 'Tuple field 6 type does not match one required by operation: expected scalar'
+...
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
+---
+- error: 'Tuple field 7 type does not match one required by operation: expected map'
+...
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
+---
+- error: Tuple field count 7 is less than required by space format or defined indexes
+    (expected at least 9)
+...
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
+---
+- error: Tuple field count 8 is less than required by space format or defined indexes
+    (expected at least 9)
+...
+s:truncate()
+---
+...
+--
+-- gh-1014: field names.
+--
+format = {}
+---
+...
+format[1] = {name = 'field1', type = 'unsigned'}
+---
+...
+format[2] = {name = 'field2'}
+---
+...
+format[3] = {name = 'field1'}
+---
+...
+s:format(format)
+---
+- error: Space field 'field1' is duplicate
+...
+s:drop()
+---
+...
+-- https://github.com/tarantool/tarantool/issues/2815
+-- Extend space format definition syntax
+format = {{name='key',type='unsigned'}, {name='value',type='string'}}
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+...
+s:format()
+---
+- [{'name': 'key', 'type': 'unsigned'}, {'name': 'value', 'type': 'string'}]
+...
+s:format({'id', 'name'})
+---
+...
+s:format()
+---
+- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'any'}]
+...
+s:format({'id', {'name1'}})
+---
+...
+s:format()
+---
+- [{'name': 'id', 'type': 'any'}, {'name': 'name1', 'type': 'any'}]
+...
+s:format({'id', {'name2', 'string'}})
+---
+...
+s:format()
+---
+- [{'name': 'id', 'type': 'any'}, {'name': 'name2', 'type': 'string'}]
+...
+s:format({'id', {'name', type = 'string'}})
+---
+...
+s:format()
+---
+- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'string'}]
+...
+s:drop()
+---
+...
+format = {'key', {'value',type='string'}}
+---
+...
+s = box.schema.space.create('test', { engine = engine, format = format })
+---
+...
+s:format()
+---
+- [{'name': 'key', 'type': 'any'}, {'name': 'value', 'type': 'string'}]
+...
+s:drop()
+---
+...
+s = box.schema.space.create('test', { engine = engine })
+---
+...
+s:create_index('test', {parts = {'test'}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test', {parts = {{'test'}}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test', {parts = {{field = 'test'}}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test', {parts = {1}}).parts
+---
+- - type: scalar
+    is_nullable: false
+    fieldno: 1
+...
+s:drop()
+---
+...
+s = box.schema.space.create('test', { engine = engine })
+---
+...
+s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
+---
+...
+s:create_index('test', {parts = {'test'}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test', {parts = {{'test'}}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test', {parts = {{field = 'test'}}})
+---
+- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
+...
+s:create_index('test1', {parts = {'test1'}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 1
+...
+s:create_index('test2', {parts = {'test2'}}).parts
+---
+- error: 'Can''t create or modify index ''test2'' in space ''test'': field type ''any''
+    is not supported'
+...
+s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 1
+...
+s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 2
+...
+s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 2
+...
+s:create_index('test6', {parts = {1, 3}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 1
+  - type: integer
+    is_nullable: false
+    fieldno: 3
+...
+s:create_index('test7', {parts = {'test1', 4}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 1
+  - type: scalar
+    is_nullable: false
+    fieldno: 4
+...
+s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
+---
+- - type: integer
+    is_nullable: false
+    fieldno: 1
+  - type: scalar
+    is_nullable: false
+    fieldno: 4
+...
+s:drop()
+---
+...
+--
+-- gh-2800: space formats checking is broken.
+--
+-- Ensure that vinyl correctly process field count change.
+s = box.schema.space.create('test', {engine = engine, field_count = 2})
+---
+...
+pk = s:create_index('pk')
+---
+...
+s:replace{1, 2}
+---
+- [1, 2]
+...
+t = box.space._space:select{s.id}[1]:totable()
+---
+...
+t[5] = 1
+---
+...
+box.space._space:replace(t)
+---
+- error: Tuple field count 2 does not match space field count 1
+...
+s:drop()
+---
+...
+-- Check field type changes.
+format = {}
+---
+...
+format[1] = {name = 'field1', type = 'unsigned'}
+---
+...
+format[2] = {name = 'field2', type = 'any'}
+---
+...
+format[3] = {name = 'field3', type = 'unsigned'}
+---
+...
+format[4] = {name = 'field4', type = 'string'}
+---
+...
+format[5] = {name = 'field5', type = 'number'}
+---
+...
+format[6] = {name = 'field6', type = 'integer'}
+---
+...
+format[7] = {name = 'field7', type = 'boolean'}
+---
+...
+format[8] = {name = 'field8', type = 'scalar'}
+---
+...
+format[9] = {name = 'field9', type = 'array'}
+---
+...
+format[10] = {name = 'field10', type = 'map'}
+---
+...
+s = box.schema.space.create('test', {engine = engine, format = format})
+---
+...
+pk = s:create_index('pk')
+---
+...
+t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
+---
+...
+inspector:cmd("setopt delimiter ';'")
+---
+- true
+...
+function fail_format_change(fieldno, new_type)
+    local old_type = format[fieldno].type
+    format[fieldno].type = new_type
+    local ok, msg = pcall(s.format, s, format)
+    format[fieldno].type = old_type
+    return msg
+end;
+---
+...
+function ok_format_change(fieldno, new_type)
+    local old_type = format[fieldno].type
+    format[fieldno].type = new_type
+    s:format(format)
+    s:delete{1}
+    format[fieldno].type = old_type
+    s:format(format)
+    s:replace(t)
+end;
+---
+...
+inspector:cmd("setopt delimiter ''");
+---
+- true
+...
+-- any --X--> unsigned
+fail_format_change(2, 'unsigned')
+---
+- 'Tuple field 2 type does not match one required by operation: expected unsigned'
+...
+-- unsigned -----> any
+ok_format_change(3, 'any')
+---
+...
+-- unsigned --X--> string
+fail_format_change(3, 'string')
+---
+- 'Tuple field 3 type does not match one required by operation: expected string'
+...
+-- unsigned -----> number
+ok_format_change(3, 'number')
+---
+...
+-- unsigned -----> integer
+ok_format_change(3, 'integer')
+---
+...
+-- unsigned -----> scalar
+ok_format_change(3, 'scalar')
+---
+...
+-- unsigned --X--> map
+fail_format_change(3, 'map')
+---
+- 'Tuple field 3 type does not match one required by operation: expected map'
+...
+-- string -----> any
+ok_format_change(4, 'any')
+---
+...
+-- string -----> scalar
+ok_format_change(4, 'scalar')
+---
+...
+-- string --X--> boolean
+fail_format_change(4, 'boolean')
+---
+- 'Tuple field 4 type does not match one required by operation: expected boolean'
+...
+-- number -----> any
+ok_format_change(5, 'any')
+---
+...
+-- number -----> scalar
+ok_format_change(5, 'scalar')
+---
+...
+-- number --X--> integer
+fail_format_change(5, 'integer')
+---
+- 'Tuple field 5 type does not match one required by operation: expected integer'
+...
+-- integer -----> any
+ok_format_change(6, 'any')
+---
+...
+-- integer -----> number
+ok_format_change(6, 'number')
+---
+...
+-- integer -----> scalar
+ok_format_change(6, 'scalar')
+---
+...
+-- integer --X--> unsigned
+fail_format_change(6, 'unsigned')
+---
+- 'Tuple field 6 type does not match one required by operation: expected unsigned'
+...
+-- boolean -----> any
+ok_format_change(7, 'any')
+---
+...
+-- boolean -----> scalar
+ok_format_change(7, 'scalar')
+---
+...
+-- boolean --X--> string
+fail_format_change(7, 'string')
+---
+- 'Tuple field 7 type does not match one required by operation: expected string'
+...
+-- scalar -----> any
+ok_format_change(8, 'any')
+---
+...
+-- scalar --X--> unsigned
+fail_format_change(8, 'unsigned')
+---
+- 'Tuple field 8 type does not match one required by operation: expected unsigned'
+...
+-- array -----> any
+ok_format_change(9, 'any')
+---
+...
+-- array --X--> scalar
+fail_format_change(9, 'scalar')
+---
+- 'Tuple field 9 type does not match one required by operation: expected scalar'
+...
+-- map -----> any
+ok_format_change(10, 'any')
+---
+...
+-- map --X--> scalar
+fail_format_change(10, 'scalar')
+---
+- 'Tuple field 10 type does not match one required by operation: expected scalar'
+...
+s:drop()
+---
+...
+-- Check new fields adding.
+format = {}
+---
+...
+s = box.schema.space.create('test', {engine = engine})
+---
+...
+format[1] = {name = 'field1', type = 'unsigned'}
+---
+...
+s:format(format) -- Ok, no indexes.
+---
+...
+pk = s:create_index('pk')
+---
+...
+format[2] = {name = 'field2', type = 'unsigned'}
+---
+...
+s:format(format) -- Ok, empty space.
+---
+...
+s:replace{1, 1}
+---
+- [1, 1]
+...
+format[2] = nil
+---
+...
+s:format(format) -- Ok, can delete fields with no checks.
+---
+...
+s:drop()
+---
+...
+s = box.schema.space.create('test', {engine = engine, format = format})
+---
+...
+pk = s:create_index('pk')
+---
+...
+sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
+---
+...
+sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
+---
+...
+sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
+---
+...
+s:replace{1, 1, 1, 1, 1}
+---
+- [1, 1, 1, 1, 1]
+...
+format[2] = {name = 'field2', type = 'unsigned'}
+---
+...
+format[3] = {name = 'field3', type = 'unsigned'}
+---
+...
+format[4] = {name = 'field4', type = 'any'}
+---
+...
+format[5] = {name = 'field5', type = 'unsigned'}
+---
+...
+-- Ok, all new fields are indexed or have type ANY, and new
+-- field_count <= old field_count.
+s:format(format)
+---
+...
+s:replace{1, 1, 1, 1, 1, 1}
+---
+- [1, 1, 1, 1, 1, 1]
+...
+format[6] = {name = 'field6', type = 'unsigned'}
+---
+...
+-- Ok, but check existing tuples for a new field[6].
+s:format(format)
+---
+...
+-- Fail, not enough fields.
+s:replace{2, 2, 2, 2, 2}
+---
+- error: Tuple field count 5 is less than required by space format or defined indexes
+    (expected at least 6)
+...
+s:replace{2, 2, 2, 2, 2, 2, 2}
+---
+- [2, 2, 2, 2, 2, 2, 2]
+...
+format[7] = {name = 'field7', type = 'unsigned'}
+---
+...
+-- Fail, the tuple {1, ... 1} is invalid for a new format.
+s:format(format)
+---
+- error: Tuple field count 6 is less than required by space format or defined indexes
+    (expected at least 7)
+...
+s:drop()
+---
+...
+--
+-- Allow to restrict space format, if corresponding restrictions
+-- already are defined in indexes.
+--
+s = box.schema.space.create('test', {engine = engine})
+---
+...
+_ = s:create_index('pk')
+---
+...
+format = {}
+---
+...
+format[1] = {name = 'field1'}
+---
+...
+s:replace{1}
+---
+- [1]
+...
+s:replace{100}
+---
+- [100]
+...
+s:replace{0}
+---
+- [0]
+...
+s:format(format)
+---
+...
+s:format()
+---
+- [{'name': 'field1', 'type': 'any'}]
+...
+format[1].type = 'unsigned'
+---
+...
+s:format(format)
+---
+...
+s:format()
+---
+- [{'name': 'field1', 'type': 'unsigned'}]
+...
+s:select()
+---
+- - [0]
+  - [1]
+  - [100]
+...
+s:drop()
+---
+...
diff --git a/test/engine/ddl.test.lua b/test/engine/ddl.test.lua
index 019e18a1..f3d68e1b 100644
--- a/test/engine/ddl.test.lua
+++ b/test/engine/ddl.test.lua
@@ -125,14 +125,14 @@ space:drop()
 -- is omited.
 --
 format = {{'field1', 'scalar'}}
-s = box.schema.create_space('test', {format = format})
+s = box.schema.space.create('test', {engine = engine, format = format})
 pk = s:create_index('pk')
 pk.parts[1].type
 s:drop()
 
 -- Ensure type 'any' to be not inherited.
 format = {{'field1'}}
-s = box.schema.create_space('test', {format = format})
+s = box.schema.space.create('test', {engine = engine, format = format})
 pk = s:create_index('pk')
 pk.parts[1].type
 s:drop()
@@ -141,7 +141,7 @@ s:drop()
 -- gh-3229: update optionality if a space format is changed too,
 -- not only when indexes are updated.
 --
-s = box.schema.create_space('test', {engine = engine})
+s = box.schema.space.create('test', {engine = engine})
 format = {}
 format[1] = {'field1', 'unsigned'}
 format[2] = {'field2', 'unsigned', is_nullable = true}
@@ -165,7 +165,7 @@ s:drop()
 --
 -- Modify key definition without index rebuild.
 --
-s = box.schema.create_space('test', {engine = engine})
+s = box.schema.space.create('test', {engine = engine})
 i1 = s:create_index('i1', {unique = true,  parts = {1, 'unsigned'}})
 i2 = s:create_index('i2', {unique = false, parts = {2, 'unsigned'}})
 i3 = s:create_index('i3', {unique = true,  parts = {3, 'unsigned'}})
@@ -195,3 +195,306 @@ i2:select()
 i3:select()
 
 s:drop()
+
+--
+-- gh-2652: validate space format.
+--
+s = box.schema.space.create('test', { engine = engine, format = "format" })
+format = { { name = 100 } }
+s = box.schema.space.create('test', { engine = engine, format = format })
+long = string.rep('a', box.schema.NAME_MAX + 1)
+format = { { name = long } }
+s = box.schema.space.create('test', { engine = engine, format = format })
+format = { { name = 'id', type = '100' } }
+s = box.schema.space.create('test', { engine = engine, format = format })
+format = { setmetatable({}, { __serialize = 'map' }) }
+s = box.schema.space.create('test', { engine = engine, format = format })
+
+-- Ensure the format is updated after index drop.
+format = { { name = 'id', type = 'unsigned' } }
+s = box.schema.space.create('test', { engine = engine, format = format })
+pk = s:create_index('pk')
+sk = s:create_index('sk', { parts = { 2, 'string' } })
+s:replace{1, 1}
+sk:drop()
+s:replace{1, 1}
+s:drop()
+
+-- Check index parts conflicting with space format.
+format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
+s = box.schema.space.create('test', { engine = engine, format = format })
+pk = s:create_index('pk')
+sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
+
+-- Check space format conflicting with index parts.
+sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
+format[2].type = 'unsigned'
+s:format(format)
+s:format()
+s.index.sk3.parts
+
+-- Space format can be updated, if conflicted index is deleted.
+sk3:drop()
+s:format(format)
+s:format()
+
+-- Check deprecated field types.
+format[2].type = 'num'
+format[3].type = 'str'
+format[4] = { name = 'field4', type = '*' }
+format
+s:format(format)
+s:format()
+s:replace{1, 2, '3', {4, 4, 4}}
+
+-- Check not indexed fields checking.
+s:truncate()
+format[2] = {name='field2', type='string'}
+format[3] = {name='field3', type='array'}
+format[4] = {name='field4', type='number'}
+format[5] = {name='field5', type='integer'}
+format[6] = {name='field6', type='scalar'}
+format[7] = {name='field7', type='map'}
+format[8] = {name='field8', type='any'}
+format[9] = {name='field9'}
+s:format(format)
+
+-- Check incorrect field types.
+format[9] = {name='err', type='any'}
+s:format(format)
+
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
+s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
+s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
+s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
+s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
+s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
+s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
+s:truncate()
+
+--
+-- gh-1014: field names.
+--
+format = {}
+format[1] = {name = 'field1', type = 'unsigned'}
+format[2] = {name = 'field2'}
+format[3] = {name = 'field1'}
+s:format(format)
+
+s:drop()
+
+-- https://github.com/tarantool/tarantool/issues/2815
+-- Extend space format definition syntax
+format = {{name='key',type='unsigned'}, {name='value',type='string'}}
+s = box.schema.space.create('test', { engine = engine, format = format })
+s:format()
+s:format({'id', 'name'})
+s:format()
+s:format({'id', {'name1'}})
+s:format()
+s:format({'id', {'name2', 'string'}})
+s:format()
+s:format({'id', {'name', type = 'string'}})
+s:format()
+s:drop()
+
+format = {'key', {'value',type='string'}}
+s = box.schema.space.create('test', { engine = engine, format = format })
+s:format()
+s:drop()
+
+s = box.schema.space.create('test', { engine = engine })
+s:create_index('test', {parts = {'test'}})
+s:create_index('test', {parts = {{'test'}}})
+s:create_index('test', {parts = {{field = 'test'}}})
+s:create_index('test', {parts = {1}}).parts
+s:drop()
+
+s = box.schema.space.create('test', { engine = engine })
+s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
+s:create_index('test', {parts = {'test'}})
+s:create_index('test', {parts = {{'test'}}})
+s:create_index('test', {parts = {{field = 'test'}}})
+s:create_index('test1', {parts = {'test1'}}).parts
+s:create_index('test2', {parts = {'test2'}}).parts
+s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
+s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
+s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
+s:create_index('test6', {parts = {1, 3}}).parts
+s:create_index('test7', {parts = {'test1', 4}}).parts
+s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
+s:drop()
+
+--
+-- gh-2800: space formats checking is broken.
+--
+
+-- Ensure that vinyl correctly process field count change.
+s = box.schema.space.create('test', {engine = engine, field_count = 2})
+pk = s:create_index('pk')
+s:replace{1, 2}
+t = box.space._space:select{s.id}[1]:totable()
+t[5] = 1
+box.space._space:replace(t)
+s:drop()
+
+-- Check field type changes.
+format = {}
+format[1] = {name = 'field1', type = 'unsigned'}
+format[2] = {name = 'field2', type = 'any'}
+format[3] = {name = 'field3', type = 'unsigned'}
+format[4] = {name = 'field4', type = 'string'}
+format[5] = {name = 'field5', type = 'number'}
+format[6] = {name = 'field6', type = 'integer'}
+format[7] = {name = 'field7', type = 'boolean'}
+format[8] = {name = 'field8', type = 'scalar'}
+format[9] = {name = 'field9', type = 'array'}
+format[10] = {name = 'field10', type = 'map'}
+s = box.schema.space.create('test', {engine = engine, format = format})
+pk = s:create_index('pk')
+t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
+
+inspector:cmd("setopt delimiter ';'")
+function fail_format_change(fieldno, new_type)
+    local old_type = format[fieldno].type
+    format[fieldno].type = new_type
+    local ok, msg = pcall(s.format, s, format)
+    format[fieldno].type = old_type
+    return msg
+end;
+
+function ok_format_change(fieldno, new_type)
+    local old_type = format[fieldno].type
+    format[fieldno].type = new_type
+    s:format(format)
+    s:delete{1}
+    format[fieldno].type = old_type
+    s:format(format)
+    s:replace(t)
+end;
+inspector:cmd("setopt delimiter ''");
+
+-- any --X--> unsigned
+fail_format_change(2, 'unsigned')
+
+-- unsigned -----> any
+ok_format_change(3, 'any')
+-- unsigned --X--> string
+fail_format_change(3, 'string')
+-- unsigned -----> number
+ok_format_change(3, 'number')
+-- unsigned -----> integer
+ok_format_change(3, 'integer')
+-- unsigned -----> scalar
+ok_format_change(3, 'scalar')
+-- unsigned --X--> map
+fail_format_change(3, 'map')
+
+-- string -----> any
+ok_format_change(4, 'any')
+-- string -----> scalar
+ok_format_change(4, 'scalar')
+-- string --X--> boolean
+fail_format_change(4, 'boolean')
+
+-- number -----> any
+ok_format_change(5, 'any')
+-- number -----> scalar
+ok_format_change(5, 'scalar')
+-- number --X--> integer
+fail_format_change(5, 'integer')
+
+-- integer -----> any
+ok_format_change(6, 'any')
+-- integer -----> number
+ok_format_change(6, 'number')
+-- integer -----> scalar
+ok_format_change(6, 'scalar')
+-- integer --X--> unsigned
+fail_format_change(6, 'unsigned')
+
+-- boolean -----> any
+ok_format_change(7, 'any')
+-- boolean -----> scalar
+ok_format_change(7, 'scalar')
+-- boolean --X--> string
+fail_format_change(7, 'string')
+
+-- scalar -----> any
+ok_format_change(8, 'any')
+-- scalar --X--> unsigned
+fail_format_change(8, 'unsigned')
+
+-- array -----> any
+ok_format_change(9, 'any')
+-- array --X--> scalar
+fail_format_change(9, 'scalar')
+
+-- map -----> any
+ok_format_change(10, 'any')
+-- map --X--> scalar
+fail_format_change(10, 'scalar')
+
+s:drop()
+
+-- Check new fields adding.
+format = {}
+s = box.schema.space.create('test', {engine = engine})
+format[1] = {name = 'field1', type = 'unsigned'}
+s:format(format) -- Ok, no indexes.
+pk = s:create_index('pk')
+format[2] = {name = 'field2', type = 'unsigned'}
+s:format(format) -- Ok, empty space.
+s:replace{1, 1}
+format[2] = nil
+s:format(format) -- Ok, can delete fields with no checks.
+s:drop()
+
+s = box.schema.space.create('test', {engine = engine, format = format})
+pk = s:create_index('pk')
+sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
+sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
+sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
+s:replace{1, 1, 1, 1, 1}
+format[2] = {name = 'field2', type = 'unsigned'}
+format[3] = {name = 'field3', type = 'unsigned'}
+format[4] = {name = 'field4', type = 'any'}
+format[5] = {name = 'field5', type = 'unsigned'}
+-- Ok, all new fields are indexed or have type ANY, and new
+-- field_count <= old field_count.
+s:format(format)
+
+s:replace{1, 1, 1, 1, 1, 1}
+format[6] = {name = 'field6', type = 'unsigned'}
+-- Ok, but check existing tuples for a new field[6].
+s:format(format)
+
+-- Fail, not enough fields.
+s:replace{2, 2, 2, 2, 2}
+
+s:replace{2, 2, 2, 2, 2, 2, 2}
+format[7] = {name = 'field7', type = 'unsigned'}
+-- Fail, the tuple {1, ... 1} is invalid for a new format.
+s:format(format)
+s:drop()
+
+--
+-- Allow to restrict space format, if corresponding restrictions
+-- already are defined in indexes.
+--
+s = box.schema.space.create('test', {engine = engine})
+_ = s:create_index('pk')
+format = {}
+format[1] = {name = 'field1'}
+s:replace{1}
+s:replace{100}
+s:replace{0}
+s:format(format)
+s:format()
+format[1].type = 'unsigned'
+s:format(format)
+s:format()
+s:select()
+s:drop()
diff --git a/test/vinyl/ddl.result b/test/vinyl/ddl.result
index 5142f0f2..4607a44e 100644
--- a/test/vinyl/ddl.result
+++ b/test/vinyl/ddl.result
@@ -696,89 +696,6 @@ index = space:create_index('test', { type = 'tree', parts = { 2, 'map' }})
 space:drop()
 ---
 ...
---
--- Allow compatible changes of a non-empty vinyl space.
---
-space = box.schema.create_space('test', { engine = 'vinyl' })
----
-...
-pk = space:create_index('primary')
----
-...
-space:replace{1}
----
-- [1]
-...
-space:replace{2}
----
-- [2]
-...
-format = {}
----
-...
-format[1] = {name = 'field1'}
----
-...
-format[2] = {name = 'field2', is_nullable = true}
----
-...
-format[3] = {name = 'field3', is_nullable = true}
----
-...
-space:format(format)
----
-...
-t1 = space:replace{3,4,5}
----
-...
-t2 = space:replace{4,5}
----
-...
-t1.field1, t1.field2, t1.field3
----
-- 3
-- 4
-- 5
-...
-t2.field1, t2.field2, t2.field3
----
-- 4
-- 5
-- null
-...
-t1 = pk:get{1}
----
-...
-t1.field1, t1.field2, t1.field3
----
-- 1
-- null
-- null
-...
-box.snapshot()
----
-- ok
-...
-t1 = pk:get{2}
----
-...
-t1.field1, t1.field2, t1.field3
----
-- 2
-- null
-- null
-...
--- Forbid incompatible change.
-format[2].is_nullable = false
----
-...
-space:format(format)
----
-- error: Vinyl does not support changing format of a non-empty space
-...
-space:drop()
----
-...
 -- gh-3019 default index options
 box.space._space:insert{512, 1, 'test', 'vinyl', 0, setmetatable({}, {__serialize = 'map'}), {}}
 ---
diff --git a/test/vinyl/ddl.test.lua b/test/vinyl/ddl.test.lua
index c4bd36bb..637a331d 100644
--- a/test/vinyl/ddl.test.lua
+++ b/test/vinyl/ddl.test.lua
@@ -260,32 +260,6 @@ index = space:create_index('test', { type = 'tree', parts = { 2, 'array' }})
 index = space:create_index('test', { type = 'tree', parts = { 2, 'map' }})
 space:drop()
 
---
--- Allow compatible changes of a non-empty vinyl space.
---
-space = box.schema.create_space('test', { engine = 'vinyl' })
-pk = space:create_index('primary')
-space:replace{1}
-space:replace{2}
-format = {}
-format[1] = {name = 'field1'}
-format[2] = {name = 'field2', is_nullable = true}
-format[3] = {name = 'field3', is_nullable = true}
-space:format(format)
-t1 = space:replace{3,4,5}
-t2 = space:replace{4,5}
-t1.field1, t1.field2, t1.field3
-t2.field1, t2.field2, t2.field3
-t1 = pk:get{1}
-t1.field1, t1.field2, t1.field3
-box.snapshot()
-t1 = pk:get{2}
-t1.field1, t1.field2, t1.field3
--- Forbid incompatible change.
-format[2].is_nullable = false
-space:format(format)
-space:drop()
-
 -- gh-3019 default index options
 box.space._space:insert{512, 1, 'test', 'vinyl', 0, setmetatable({}, {__serialize = 'map'}), {}}
 box.space._index:insert{512, 0, 'pk', 'tree', {unique = true}, {{0, 'unsigned'}}}
diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result
index 91351086..fd21f7bb 100644
--- a/test/vinyl/errinj.result
+++ b/test/vinyl/errinj.result
@@ -1320,3 +1320,78 @@ ret
 s:drop()
 ---
 ...
+--
+-- Check that ALTER is abroted if a tuple inserted during space
+-- format change does not conform to the new format.
+--
+format = {}
+---
+...
+format[1] = {name = 'field1', type = 'unsigned'}
+---
+...
+format[2] = {name = 'field2', type = 'string', is_nullable = true}
+---
+...
+s = box.schema.space.create('test', {engine = 'vinyl', format = format})
+---
+...
+_ = s:create_index('pk', {page_size = 16})
+---
+...
+pad = string.rep('x', 16)
+---
+...
+for i = 101, 200 do s:replace{i, pad} end
+---
+...
+box.snapshot()
+---
+- ok
+...
+ch = fiber.channel(1)
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+_ = fiber.create(function()
+    fiber.sleep(0.01)
+    for i = 1, 100 do
+        s:replace{i, box.NULL}
+    end
+    ch:put(true)
+end);
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0.001)
+---
+- ok
+...
+format[2].is_nullable = false
+---
+...
+s:format(format) -- must fail
+---
+- error: 'Tuple field 2 type does not match one required by operation: expected string'
+...
+errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0)
+---
+- ok
+...
+ch:get()
+---
+- true
+...
+s:count() -- 200
+---
+- 200
+...
+s:drop()
+---
+...
diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua
index 9724a69b..64d04c62 100644
--- a/test/vinyl/errinj.test.lua
+++ b/test/vinyl/errinj.test.lua
@@ -513,3 +513,38 @@ errinj.set("ERRINJ_VY_DELAY_PK_LOOKUP", false)
 while ret == nil do fiber.sleep(0.01) end
 ret
 s:drop()
+
+--
+-- Check that ALTER is abroted if a tuple inserted during space
+-- format change does not conform to the new format.
+--
+format = {}
+format[1] = {name = 'field1', type = 'unsigned'}
+format[2] = {name = 'field2', type = 'string', is_nullable = true}
+s = box.schema.space.create('test', {engine = 'vinyl', format = format})
+_ = s:create_index('pk', {page_size = 16})
+
+pad = string.rep('x', 16)
+for i = 101, 200 do s:replace{i, pad} end
+box.snapshot()
+
+ch = fiber.channel(1)
+test_run:cmd("setopt delimiter ';'")
+_ = fiber.create(function()
+    fiber.sleep(0.01)
+    for i = 1, 100 do
+        s:replace{i, box.NULL}
+    end
+    ch:put(true)
+end);
+test_run:cmd("setopt delimiter ''");
+
+errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0.001)
+format[2].is_nullable = false
+s:format(format) -- must fail
+errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0)
+
+ch:get()
+
+s:count() -- 200
+s:drop()
-- 
2.11.0

Re: [PATCH 12/12] vinyl: allow to modify format of non-empty spaces

[Vladimir Davydov]

On Sat, Apr 07, 2018 at 04:38:09PM +0300, Vladimir Davydov wrote:
>  static int
>  vinyl_space_check_format(struct space *space, struct tuple_format *format)
>  {
> -	(void)format;
>  	struct vy_env *env = vy_env(space->engine);
> -	if (space->index_count == 0)
> +
> +	/*
> +	 * If this is local recovery, the space was checked before
> +	 * restart so there's nothing we need to do.
> +	 */
> +	if (env->status == VINYL_INITIAL_RECOVERY_LOCAL ||
> +	    env->status == VINYL_FINAL_RECOVERY_LOCAL)
>  		return 0;
> +
> +	if (space->index_count == 0)
> +		return 0; /* space is empty, nothing to do */
> +
> +	/*
> +	 * Iterate over all tuples stored in the given space and
> +	 * check each of them for conformity to the new format.
> +	 * Since read iterator may yield, we install an on_replace
> +	 * trigger to check tuples inserted after we started the
> +	 * iteration.
> +	 */
>  	struct vy_lsm *pk = vy_lsm(space->index[0]);
> -	if (env->status != VINYL_ONLINE)
> -		return 0;
> -	if (pk->stat.disk.count.rows == 0 && pk->stat.memory.count.rows == 0)
> -		return 0;
> -	diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
> -		 "changing format of a non-empty space");
> -	return -1;
> +
> +	struct tuple *key = vy_stmt_new_select(pk->env->key_format, NULL, 0);
> +	if (key == NULL)
> +		return -1;
> +
> +	struct trigger on_replace;
> +	struct vy_check_format_ctx ctx;
> +	ctx.format = format;
> +	ctx.is_failed = false;
> +	diag_create(&ctx.diag);
> +	trigger_create(&on_replace, vy_check_format_on_replace, &ctx, NULL);
> +	trigger_add(&space->on_replace, &on_replace);
> +
> +	struct vy_read_iterator itr;
> +	vy_read_iterator_open(&itr, pk, NULL, ITER_ALL, key,
> +			      &env->xm->p_global_read_view);
> +	int rc;
> +	struct tuple *tuple;
> +	while ((rc = vy_read_iterator_next(&itr, &tuple)) == 0) {
> +		if (tuple == NULL)
> +			break;

> +		if (ctx.is_failed) {
> +			diag_move(&ctx.diag, diag_get());
> +			rc = -1;
> +			break;
> +		}

This check should be before the EOF check (tuple == NULL), because even
in case read iterator returned NULL it may have yield, which might have
opened a time window for a concurrent fiber to insert a new tuple that
doesn't match the new format. Fixed on the branch. The incremental diff
is in the end of this email.

> +		rc = tuple_validate(format, tuple);
> +		if (rc != 0)
> +			break;
> +	}
> +	vy_read_iterator_close(&itr);
> +	diag_destroy(&ctx.diag);
> +	trigger_clear(&on_replace);
> +	tuple_unref(key);
> +	return rc;
>  }


diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index c2769e6d..6bed09da 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1104,13 +1104,13 @@ vinyl_space_check_format(struct space *space, struct tuple_format *format)
 	int rc;
 	struct tuple *tuple;
 	while ((rc = vy_read_iterator_next(&itr, &tuple)) == 0) {
-		if (tuple == NULL)
-			break;
 		if (ctx.is_failed) {
 			diag_move(&ctx.diag, diag_get());
 			rc = -1;
 			break;
 		}
+		if (tuple == NULL)
+			break;
 		rc = tuple_validate(format, tuple);
 		if (rc != 0)
 			break;

Re: [PATCH 12/12] vinyl: allow to modify format of non-empty spaces

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/04/09 10:33]:
> This patch implements space_vtab::check_space_format callback for vinyl
> spaces. The callback iterates over all tuples stored in the primary
> index and validates them against the new format. Since vinyl read
> iterator may yield, this callback also installs an on_replace trigger
> for the altered space. The trigger checks that tuples inserted during
> ALTER conform to the new format and aborts ALTER if they do not.
> 
> To test the feature, this patch enables memtx space format tests for
> vinyl spaces by moving them from box/alter to engine/ddl. It adds just
> one new vinyl-specific test case to vinyl/ddl. The test case checks
> that tuples inserted during ALTER are validated against the new format.

The patch is OK to push. Please consider doing all vinyl alter in
vinyl worker threads, a major compaction is IMHO an OK way to
achieve the same.

Please check out one comment below.

> ---
>  src/box/vinyl.c            |  95 +++++-
>  test/box/alter.result      | 810 +--------------------------------------------
>  test/box/alter.test.lua    | 319 ------------------
>  test/engine/ddl.result     | 787 ++++++++++++++++++++++++++++++++++++++++++-
>  test/engine/ddl.test.lua   | 311 ++++++++++++++++-
>  test/vinyl/ddl.result      |  83 -----
>  test/vinyl/ddl.test.lua    |  26 --
>  test/vinyl/errinj.result   |  75 +++++
>  test/vinyl/errinj.test.lua |  35 ++
>  9 files changed, 1288 insertions(+), 1253 deletions(-)
> 
> diff --git a/src/box/vinyl.c b/src/box/vinyl.c
> index c99b518e..c2769e6d 100644
> --- a/src/box/vinyl.c
> +++ b/src/box/vinyl.c
> @@ -1028,21 +1028,98 @@ vinyl_space_prepare_alter(struct space *old_space, struct space *new_space)
>  	return 0;
>  }
>  
> +/** Argument passed to vy_check_format_on_replace(). */
> +struct vy_check_format_ctx {
> +	/** Format to check new tuples against. */
> +	struct tuple_format *format;
> +	/** Set if a new tuple doesn't conform to the format. */
> +	bool is_failed;
> +	/** Container for storing errors. */
> +	struct diag diag;
> +};
> +
> +/**
> + * This is an on_replace trigger callback that checks inserted
> + * tuples against a new format.
> + */
> +static void
> +vy_check_format_on_replace(struct trigger *trigger, void *event)
> +{
> +	struct txn *txn = event;
> +	struct txn_stmt *stmt = txn_current_stmt(txn);
> +	struct vy_check_format_ctx *ctx = trigger->data;
> +
> +	if (stmt->new_tuple == NULL)
> +		return; /* DELETE, nothing to do */
> +
> +	if (ctx->is_failed)
> +		return; /* already failed, nothing to do */
> +
> +	if (tuple_validate(ctx->format, stmt->new_tuple) != 0) {
> +		ctx->is_failed = true;
> +		diag_move(diag_get(), &ctx->diag);
> +	}
> +}
> +
>  static int
>  vinyl_space_check_format(struct space *space, struct tuple_format *format)
>  {
> -	(void)format;
>  	struct vy_env *env = vy_env(space->engine);
> -	if (space->index_count == 0)
> +
> +	/*
> +	 * If this is local recovery, the space was checked before
> +	 * restart so there's nothing we need to do.
> +	 */
> +	if (env->status == VINYL_INITIAL_RECOVERY_LOCAL ||
> +	    env->status == VINYL_FINAL_RECOVERY_LOCAL)
>  		return 0;
> +
> +	if (space->index_count == 0)
> +		return 0; /* space is empty, nothing to do */
> +
> +	/*
> +	 * Iterate over all tuples stored in the given space and
> +	 * check each of them for conformity to the new format.
> +	 * Since read iterator may yield, we install an on_replace
> +	 * trigger to check tuples inserted after we started the
> +	 * iteration.
> +	 */
>  	struct vy_lsm *pk = vy_lsm(space->index[0]);
> -	if (env->status != VINYL_ONLINE)
> -		return 0;
> -	if (pk->stat.disk.count.rows == 0 && pk->stat.memory.count.rows == 0)
> -		return 0;
> -	diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
> -		 "changing format of a non-empty space");
> -	return -1;
> +
> +	struct tuple *key = vy_stmt_new_select(pk->env->key_format, NULL, 0);
> +	if (key == NULL)
> +		return -1;
> +
> +	struct trigger on_replace;
> +	struct vy_check_format_ctx ctx;
> +	ctx.format = format;
> +	ctx.is_failed = false;
> +	diag_create(&ctx.diag);
> +	trigger_create(&on_replace, vy_check_format_on_replace, &ctx, NULL);
> +	trigger_add(&space->on_replace, &on_replace);
> +
> +	struct vy_read_iterator itr;
> +	vy_read_iterator_open(&itr, pk, NULL, ITER_ALL, key,
> +			      &env->xm->p_global_read_view);
> +	int rc;
> +	struct tuple *tuple;
> +	while ((rc = vy_read_iterator_next(&itr, &tuple)) == 0) {
> +		if (tuple == NULL)
> +			break;
> +		if (ctx.is_failed) {
> +			diag_move(&ctx.diag, diag_get());
> +			rc = -1;
> +			break;
> +		}
> +		rc = tuple_validate(format, tuple);
> +		if (rc != 0)
> +			break;
> +	}

Please don't assume this yields naturally - it won't in a whole
bunch of critical cases. Better yet to move it to compaction
thread and perform the check along with compaction.

> +	vy_read_iterator_close(&itr);
> +	diag_destroy(&ctx.diag);
> +	trigger_clear(&on_replace);
> +	tuple_unref(key);
> +	return rc;
>  }
>  
>  static void
> diff --git a/test/box/alter.result b/test/box/alter.result
> index 945b0cfd..a78dd3e9 100644
> --- a/test/box/alter.result
> +++ b/test/box/alter.result
> @@ -817,753 +817,6 @@ ts:drop()
>  ---
>  ...
>  --
> --- gh-2652: validate space format.
> ---
> -s = box.schema.space.create('test', { format = "format" })
> ----
> -- error: Illegal parameters, options parameter 'format' should be of type table
> -...
> -format = { { name = 100 } }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -- error: 'Illegal parameters, format[1]: name (string) is expected'
> -...
> -long = string.rep('a', box.schema.NAME_MAX + 1)
> ----
> -...
> -format = { { name = long } }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -- error: 'Failed to create space ''test'': field 1 name is too long'
> -...
> -format = { { name = 'id', type = '100' } }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -- error: 'Failed to create space ''test'': field 1 has unknown field type'
> -...
> -format = { utils.setmap({}) }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -- error: 'Illegal parameters, format[1]: name (string) is expected'
> -...
> --- Ensure the format is updated after index drop.
> -format = { { name = 'id', type = 'unsigned' } }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -sk = s:create_index('sk', { parts = { 2, 'string' } })
> ----
> -...
> -s:replace{1, 1}
> ----
> -- error: 'Tuple field 2 type does not match one required by operation: expected string'
> -...
> -sk:drop()
> ----
> -...
> -s:replace{1, 1}
> ----
> -- [1, 1]
> -...
> -s:drop()
> ----
> -...
> --- Check index parts conflicting with space format.
> -format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
> ----
> -- error: Field 'field2' has type 'string' in space format, but type 'unsigned' in
> -    index definition
> -...
> --- Check space format conflicting with index parts.
> -sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
> ----
> -...
> -format[2].type = 'unsigned'
> ----
> -...
> -s:format(format)
> ----
> -- error: Field 'field2' has type 'unsigned' in space format, but type 'string' in
> -    index definition
> -...
> -s:format()
> ----
> -- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'string'}, {
> -    'name': 'field3', 'type': 'scalar'}]
> -...
> -s.index.sk3.parts
> ----
> -- - type: string
> -    is_nullable: false
> -    fieldno: 2
> -...
> --- Space format can be updated, if conflicted index is deleted.
> -sk3:drop()
> ----
> -...
> -s:format(format)
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'unsigned'},
> -  {'name': 'field3', 'type': 'scalar'}]
> -...
> --- Check deprecated field types.
> -format[2].type = 'num'
> ----
> -...
> -format[3].type = 'str'
> ----
> -...
> -format[4] = { name = 'field4', type = '*' }
> ----
> -...
> -format
> ----
> -- - name: field1
> -    type: unsigned
> -  - name: field2
> -    type: num
> -  - name: field3
> -    type: str
> -  - name: field4
> -    type: '*'
> -...
> -s:format(format)
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'num'}, {'name': 'field3',
> -    'type': 'str'}, {'name': 'field4', 'type': '*'}]
> -...
> -s:replace{1, 2, '3', {4, 4, 4}}
> ----
> -- [1, 2, '3', [4, 4, 4]]
> -...
> --- Check not indexed fields checking.
> -s:truncate()
> ----
> -...
> -format[2] = {name='field2', type='string'}
> ----
> -...
> -format[3] = {name='field3', type='array'}
> ----
> -...
> -format[4] = {name='field4', type='number'}
> ----
> -...
> -format[5] = {name='field5', type='integer'}
> ----
> -...
> -format[6] = {name='field6', type='scalar'}
> ----
> -...
> -format[7] = {name='field7', type='map'}
> ----
> -...
> -format[8] = {name='field8', type='any'}
> ----
> -...
> -format[9] = {name='field9'}
> ----
> -...
> -s:format(format)
> ----
> -...
> --- Check incorrect field types.
> -format[9] = {name='err', type='any'}
> ----
> -...
> -s:format(format)
> ----
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> ----
> -- [1, '2', [3, 3], 4.4, -5, true, {'value': 7}, 8, 9]
> -...
> -s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> ----
> -- error: 'Tuple field 2 type does not match one required by operation: expected string'
> -...
> -s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
> ----
> -- error: 'Tuple field 3 type does not match one required by operation: expected array'
> -...
> -s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
> ----
> -- error: 'Tuple field 4 type does not match one required by operation: expected number'
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
> ----
> -- error: 'Tuple field 5 type does not match one required by operation: expected integer'
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
> ----
> -- error: 'Tuple field 6 type does not match one required by operation: expected scalar'
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
> ----
> -- error: 'Tuple field 7 type does not match one required by operation: expected map'
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
> ----
> -- error: Tuple field count 7 is less than required by space format or defined indexes
> -    (expected at least 9)
> -...
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
> ----
> -- error: Tuple field count 8 is less than required by space format or defined indexes
> -    (expected at least 9)
> -...
> -s:truncate()
> ----
> -...
> ---
> --- gh-1014: field names.
> ---
> -format = {}
> ----
> -...
> -format[1] = {name = 'field1', type = 'unsigned'}
> ----
> -...
> -format[2] = {name = 'field2'}
> ----
> -...
> -format[3] = {name = 'field1'}
> ----
> -...
> -s:format(format)
> ----
> -- error: Space field 'field1' is duplicate
> -...
> -s:drop()
> ----
> -...
> --- https://github.com/tarantool/tarantool/issues/2815
> --- Extend space format definition syntax
> -format = {{name='key',type='unsigned'}, {name='value',type='string'}}
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'key', 'type': 'unsigned'}, {'name': 'value', 'type': 'string'}]
> -...
> -s:format({'id', 'name'})
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'any'}]
> -...
> -s:format({'id', {'name1'}})
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'id', 'type': 'any'}, {'name': 'name1', 'type': 'any'}]
> -...
> -s:format({'id', {'name2', 'string'}})
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'id', 'type': 'any'}, {'name': 'name2', 'type': 'string'}]
> -...
> -s:format({'id', {'name', type = 'string'}})
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'string'}]
> -...
> -s:drop()
> ----
> -...
> -format = {'key', {'value',type='string'}}
> ----
> -...
> -s = box.schema.space.create('test', { format = format })
> ----
> -...
> -s:format()
> ----
> -- [{'name': 'key', 'type': 'any'}, {'name': 'value', 'type': 'string'}]
> -...
> -s:drop()
> ----
> -...
> -s = box.schema.space.create('test')
> ----
> -...
> -s:create_index('test', {parts = {'test'}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test', {parts = {{'test'}}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test', {parts = {{field = 'test'}}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test', {parts = {1}}).parts
> ----
> -- - type: scalar
> -    is_nullable: false
> -    fieldno: 1
> -...
> -s:drop()
> ----
> -...
> -s = box.schema.space.create('test')
> ----
> -...
> -s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
> ----
> -...
> -s:create_index('test', {parts = {'test'}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test', {parts = {{'test'}}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test', {parts = {{field = 'test'}}})
> ----
> -- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> -...
> -s:create_index('test1', {parts = {'test1'}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 1
> -...
> -s:create_index('test2', {parts = {'test2'}}).parts
> ----
> -- error: 'Can''t create or modify index ''test2'' in space ''test'': field type ''any''
> -    is not supported'
> -...
> -s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 1
> -...
> -s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 2
> -...
> -s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 2
> -...
> -s:create_index('test6', {parts = {1, 3}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 1
> -  - type: integer
> -    is_nullable: false
> -    fieldno: 3
> -...
> -s:create_index('test7', {parts = {'test1', 4}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 1
> -  - type: scalar
> -    is_nullable: false
> -    fieldno: 4
> -...
> -s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
> ----
> -- - type: integer
> -    is_nullable: false
> -    fieldno: 1
> -  - type: scalar
> -    is_nullable: false
> -    fieldno: 4
> -...
> -s:drop()
> ----
> -...
> ---
> --- gh-2800: space formats checking is broken.
> ---
> --- Ensure that vinyl correctly process field count change.
> -s = box.schema.space.create('test', {engine = 'vinyl', field_count = 2})
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -s:replace{1, 2}
> ----
> -- [1, 2]
> -...
> -t = box.space._space:select{s.id}[1]:totable()
> ----
> -...
> -t[5] = 1
> ----
> -...
> -box.space._space:replace(t)
> ----
> -- error: Vinyl does not support changing format of a non-empty space
> -...
> -s:drop()
> ----
> -...
> --- Check field type changes.
> -format = {}
> ----
> -...
> -format[1] = {name = 'field1', type = 'unsigned'}
> ----
> -...
> -format[2] = {name = 'field2', type = 'any'}
> ----
> -...
> -format[3] = {name = 'field3', type = 'unsigned'}
> ----
> -...
> -format[4] = {name = 'field4', type = 'string'}
> ----
> -...
> -format[5] = {name = 'field5', type = 'number'}
> ----
> -...
> -format[6] = {name = 'field6', type = 'integer'}
> ----
> -...
> -format[7] = {name = 'field7', type = 'boolean'}
> ----
> -...
> -format[8] = {name = 'field8', type = 'scalar'}
> ----
> -...
> -format[9] = {name = 'field9', type = 'array'}
> ----
> -...
> -format[10] = {name = 'field10', type = 'map'}
> ----
> -...
> -s = box.schema.space.create('test', {format = format})
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
> ----
> -...
> -test_run:cmd("setopt delimiter ';'")
> ----
> -- true
> -...
> -function fail_format_change(fieldno, new_type)
> -    local old_type = format[fieldno].type
> -    format[fieldno].type = new_type
> -    local ok, msg = pcall(s.format, s, format)
> -    format[fieldno].type = old_type
> -    return msg
> -end;
> ----
> -...
> -function ok_format_change(fieldno, new_type)
> -    local old_type = format[fieldno].type
> -    format[fieldno].type = new_type
> -    s:format(format)
> -    s:delete{1}
> -    format[fieldno].type = old_type
> -    s:format(format)
> -    s:replace(t)
> -end;
> ----
> -...
> -test_run:cmd("setopt delimiter ''");
> ----
> -- true
> -...
> --- any --X--> unsigned
> -fail_format_change(2, 'unsigned')
> ----
> -- 'Tuple field 2 type does not match one required by operation: expected unsigned'
> -...
> --- unsigned -----> any
> -ok_format_change(3, 'any')
> ----
> -...
> --- unsigned --X--> string
> -fail_format_change(3, 'string')
> ----
> -- 'Tuple field 3 type does not match one required by operation: expected string'
> -...
> --- unsigned -----> number
> -ok_format_change(3, 'number')
> ----
> -...
> --- unsigned -----> integer
> -ok_format_change(3, 'integer')
> ----
> -...
> --- unsigned -----> scalar
> -ok_format_change(3, 'scalar')
> ----
> -...
> --- unsigned --X--> map
> -fail_format_change(3, 'map')
> ----
> -- 'Tuple field 3 type does not match one required by operation: expected map'
> -...
> --- string -----> any
> -ok_format_change(4, 'any')
> ----
> -...
> --- string -----> scalar
> -ok_format_change(4, 'scalar')
> ----
> -...
> --- string --X--> boolean
> -fail_format_change(4, 'boolean')
> ----
> -- 'Tuple field 4 type does not match one required by operation: expected boolean'
> -...
> --- number -----> any
> -ok_format_change(5, 'any')
> ----
> -...
> --- number -----> scalar
> -ok_format_change(5, 'scalar')
> ----
> -...
> --- number --X--> integer
> -fail_format_change(5, 'integer')
> ----
> -- 'Tuple field 5 type does not match one required by operation: expected integer'
> -...
> --- integer -----> any
> -ok_format_change(6, 'any')
> ----
> -...
> --- integer -----> number
> -ok_format_change(6, 'number')
> ----
> -...
> --- integer -----> scalar
> -ok_format_change(6, 'scalar')
> ----
> -...
> --- integer --X--> unsigned
> -fail_format_change(6, 'unsigned')
> ----
> -- 'Tuple field 6 type does not match one required by operation: expected unsigned'
> -...
> --- boolean -----> any
> -ok_format_change(7, 'any')
> ----
> -...
> --- boolean -----> scalar
> -ok_format_change(7, 'scalar')
> ----
> -...
> --- boolean --X--> string
> -fail_format_change(7, 'string')
> ----
> -- 'Tuple field 7 type does not match one required by operation: expected string'
> -...
> --- scalar -----> any
> -ok_format_change(8, 'any')
> ----
> -...
> --- scalar --X--> unsigned
> -fail_format_change(8, 'unsigned')
> ----
> -- 'Tuple field 8 type does not match one required by operation: expected unsigned'
> -...
> --- array -----> any
> -ok_format_change(9, 'any')
> ----
> -...
> --- array --X--> scalar
> -fail_format_change(9, 'scalar')
> ----
> -- 'Tuple field 9 type does not match one required by operation: expected scalar'
> -...
> --- map -----> any
> -ok_format_change(10, 'any')
> ----
> -...
> --- map --X--> scalar
> -fail_format_change(10, 'scalar')
> ----
> -- 'Tuple field 10 type does not match one required by operation: expected scalar'
> -...
> -s:drop()
> ----
> -...
> --- Check new fields adding.
> -format = {}
> ----
> -...
> -s = box.schema.space.create('test')
> ----
> -...
> -format[1] = {name = 'field1', type = 'unsigned'}
> ----
> -...
> -s:format(format) -- Ok, no indexes.
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -format[2] = {name = 'field2', type = 'unsigned'}
> ----
> -...
> -s:format(format) -- Ok, empty space.
> ----
> -...
> -s:replace{1, 1}
> ----
> -- [1, 1]
> -...
> -format[2] = nil
> ----
> -...
> -s:format(format) -- Ok, can delete fields with no checks.
> ----
> -...
> -s:delete{1}
> ----
> -- [1, 1]
> -...
> -sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
> ----
> -...
> -sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
> ----
> -...
> -sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
> ----
> -...
> -s:replace{1, 1, 1, 1, 1}
> ----
> -- [1, 1, 1, 1, 1]
> -...
> -format[2] = {name = 'field2', type = 'unsigned'}
> ----
> -...
> -format[3] = {name = 'field3', type = 'unsigned'}
> ----
> -...
> -format[4] = {name = 'field4', type = 'any'}
> ----
> -...
> -format[5] = {name = 'field5', type = 'unsigned'}
> ----
> -...
> --- Ok, all new fields are indexed or have type ANY, and new
> --- field_count <= old field_count.
> -s:format(format)
> ----
> -...
> -s:replace{1, 1, 1, 1, 1, 1}
> ----
> -- [1, 1, 1, 1, 1, 1]
> -...
> -format[6] = {name = 'field6', type = 'unsigned'}
> ----
> -...
> --- Ok, but check existing tuples for a new field[6].
> -s:format(format)
> ----
> -...
> --- Fail, not enough fields.
> -s:replace{2, 2, 2, 2, 2}
> ----
> -- error: Tuple field count 5 is less than required by space format or defined indexes
> -    (expected at least 6)
> -...
> -s:replace{2, 2, 2, 2, 2, 2, 2}
> ----
> -- [2, 2, 2, 2, 2, 2, 2]
> -...
> -format[7] = {name = 'field7', type = 'unsigned'}
> ----
> -...
> --- Fail, the tuple {1, ... 1} is invalid for a new format.
> -s:format(format)
> ----
> -- error: Tuple field count 6 is less than required by space format or defined indexes
> -    (expected at least 7)
> -...
> -s:drop()
> ----
> -...
> --- Vinyl does not support adding fields to a not empty space.
> -s = box.schema.space.create('test', {engine = 'vinyl'})
> ----
> -...
> -pk = s:create_index('pk')
> ----
> -...
> -s:replace{1,1}
> ----
> -- [1, 1]
> -...
> -format = {}
> ----
> -...
> -format[1] = {name = 'field1', type = 'unsigned'}
> ----
> -...
> -format[2] = {name = 'field2', type = 'unsigned'}
> ----
> -...
> -s:format(format)
> ----
> -- error: Vinyl does not support changing format of a non-empty space
> -...
> -s:drop()
> ----
> -...
> ---
>  -- gh-1557: NULL in indexes.
>  --
>  NULL = require('msgpack').NULL
> @@ -1607,7 +860,7 @@ s:create_index('primary', { parts = {'field1'} })
>      is_nullable: false
>      fieldno: 1
>    id: 0
> -  space_id: 747
> +  space_id: 733
>    name: primary
>    type: TREE
>  ...
> @@ -1684,7 +937,7 @@ s:create_index('secondary', { parts = {{2, 'string', is_nullable = true}} })
>      is_nullable: true
>      fieldno: 2
>    id: 1
> -  space_id: 747
> +  space_id: 733
>    name: secondary
>    type: TREE
>  ...
> @@ -1804,65 +1057,6 @@ s:drop()
>  ---
>  ...
>  --
> --- Allow to restrict space format, if corresponding restrictions
> --- already are defined in indexes.
> ---
> -test_run:cmd("setopt delimiter ';'")
> ----
> -- true
> -...
> -function check_format_restriction(engine, name)
> -    local s = box.schema.create_space(name, {engine = engine})
> -    local pk = s:create_index('pk')
> -    local format = {}
> -    format[1] = {name = 'field1'}
> -    s:replace{1}
> -    s:replace{100}
> -    s:replace{0}
> -    s:format(format)
> -    s:format()
> -    format[1].type = 'unsigned'
> -    s:format(format)
> -end;
> ----
> -...
> -test_run:cmd("setopt delimiter ''");
> ----
> -- true
> -...
> -check_format_restriction('memtx', 'test1')
> ----
> -...
> -check_format_restriction('vinyl', 'test2')
> ----
> -...
> -box.space.test1:format()
> ----
> -- [{'name': 'field1', 'type': 'unsigned'}]
> -...
> -box.space.test1:select{}
> ----
> -- - [0]
> -  - [1]
> -  - [100]
> -...
> -box.space.test2:format()
> ----
> -- [{'name': 'field1', 'type': 'unsigned'}]
> -...
> -box.space.test2:select{}
> ----
> -- - [0]
> -  - [1]
> -  - [100]
> -...
> -box.space.test1:drop()
> ----
> -...
> -box.space.test2:drop()
> ----
> -...
> ---
>  -- Allow to change is_nullable in index definition on non-empty
>  -- space.
>  --
> diff --git a/test/box/alter.test.lua b/test/box/alter.test.lua
> index f6b2eb49..ab713584 100644
> --- a/test/box/alter.test.lua
> +++ b/test/box/alter.test.lua
> @@ -316,297 +316,6 @@ n
>  ts:drop()
>  
>  --
> --- gh-2652: validate space format.
> ---
> -s = box.schema.space.create('test', { format = "format" })
> -format = { { name = 100 } }
> -s = box.schema.space.create('test', { format = format })
> -long = string.rep('a', box.schema.NAME_MAX + 1)
> -format = { { name = long } }
> -s = box.schema.space.create('test', { format = format })
> -format = { { name = 'id', type = '100' } }
> -s = box.schema.space.create('test', { format = format })
> -format = { utils.setmap({}) }
> -s = box.schema.space.create('test', { format = format })
> -
> --- Ensure the format is updated after index drop.
> -format = { { name = 'id', type = 'unsigned' } }
> -s = box.schema.space.create('test', { format = format })
> -pk = s:create_index('pk')
> -sk = s:create_index('sk', { parts = { 2, 'string' } })
> -s:replace{1, 1}
> -sk:drop()
> -s:replace{1, 1}
> -s:drop()
> -
> --- Check index parts conflicting with space format.
> -format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
> -s = box.schema.space.create('test', { format = format })
> -pk = s:create_index('pk')
> -sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
> -
> --- Check space format conflicting with index parts.
> -sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
> -format[2].type = 'unsigned'
> -s:format(format)
> -s:format()
> -s.index.sk3.parts
> -
> --- Space format can be updated, if conflicted index is deleted.
> -sk3:drop()
> -s:format(format)
> -s:format()
> -
> --- Check deprecated field types.
> -format[2].type = 'num'
> -format[3].type = 'str'
> -format[4] = { name = 'field4', type = '*' }
> -format
> -s:format(format)
> -s:format()
> -s:replace{1, 2, '3', {4, 4, 4}}
> -
> --- Check not indexed fields checking.
> -s:truncate()
> -format[2] = {name='field2', type='string'}
> -format[3] = {name='field3', type='array'}
> -format[4] = {name='field4', type='number'}
> -format[5] = {name='field5', type='integer'}
> -format[6] = {name='field6', type='scalar'}
> -format[7] = {name='field7', type='map'}
> -format[8] = {name='field8', type='any'}
> -format[9] = {name='field9'}
> -s:format(format)
> -
> --- Check incorrect field types.
> -format[9] = {name='err', type='any'}
> -s:format(format)
> -
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> -s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> -s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
> -s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
> -s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
> -s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
> -s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
> -s:truncate()
> -
> ---
> --- gh-1014: field names.
> ---
> -format = {}
> -format[1] = {name = 'field1', type = 'unsigned'}
> -format[2] = {name = 'field2'}
> -format[3] = {name = 'field1'}
> -s:format(format)
> -
> -s:drop()
> -
> --- https://github.com/tarantool/tarantool/issues/2815
> --- Extend space format definition syntax
> -format = {{name='key',type='unsigned'}, {name='value',type='string'}}
> -s = box.schema.space.create('test', { format = format })
> -s:format()
> -s:format({'id', 'name'})
> -s:format()
> -s:format({'id', {'name1'}})
> -s:format()
> -s:format({'id', {'name2', 'string'}})
> -s:format()
> -s:format({'id', {'name', type = 'string'}})
> -s:format()
> -s:drop()
> -
> -format = {'key', {'value',type='string'}}
> -s = box.schema.space.create('test', { format = format })
> -s:format()
> -s:drop()
> -
> -s = box.schema.space.create('test')
> -s:create_index('test', {parts = {'test'}})
> -s:create_index('test', {parts = {{'test'}}})
> -s:create_index('test', {parts = {{field = 'test'}}})
> -s:create_index('test', {parts = {1}}).parts
> -s:drop()
> -
> -s = box.schema.space.create('test')
> -s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
> -s:create_index('test', {parts = {'test'}})
> -s:create_index('test', {parts = {{'test'}}})
> -s:create_index('test', {parts = {{field = 'test'}}})
> -s:create_index('test1', {parts = {'test1'}}).parts
> -s:create_index('test2', {parts = {'test2'}}).parts
> -s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
> -s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
> -s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
> -s:create_index('test6', {parts = {1, 3}}).parts
> -s:create_index('test7', {parts = {'test1', 4}}).parts
> -s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
> -s:drop()
> -
> ---
> --- gh-2800: space formats checking is broken.
> ---
> -
> --- Ensure that vinyl correctly process field count change.
> -s = box.schema.space.create('test', {engine = 'vinyl', field_count = 2})
> -pk = s:create_index('pk')
> -s:replace{1, 2}
> -t = box.space._space:select{s.id}[1]:totable()
> -t[5] = 1
> -box.space._space:replace(t)
> -s:drop()
> -
> --- Check field type changes.
> -format = {}
> -format[1] = {name = 'field1', type = 'unsigned'}
> -format[2] = {name = 'field2', type = 'any'}
> -format[3] = {name = 'field3', type = 'unsigned'}
> -format[4] = {name = 'field4', type = 'string'}
> -format[5] = {name = 'field5', type = 'number'}
> -format[6] = {name = 'field6', type = 'integer'}
> -format[7] = {name = 'field7', type = 'boolean'}
> -format[8] = {name = 'field8', type = 'scalar'}
> -format[9] = {name = 'field9', type = 'array'}
> -format[10] = {name = 'field10', type = 'map'}
> -s = box.schema.space.create('test', {format = format})
> -pk = s:create_index('pk')
> -t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
> -
> -test_run:cmd("setopt delimiter ';'")
> -function fail_format_change(fieldno, new_type)
> -    local old_type = format[fieldno].type
> -    format[fieldno].type = new_type
> -    local ok, msg = pcall(s.format, s, format)
> -    format[fieldno].type = old_type
> -    return msg
> -end;
> -
> -function ok_format_change(fieldno, new_type)
> -    local old_type = format[fieldno].type
> -    format[fieldno].type = new_type
> -    s:format(format)
> -    s:delete{1}
> -    format[fieldno].type = old_type
> -    s:format(format)
> -    s:replace(t)
> -end;
> -test_run:cmd("setopt delimiter ''");
> -
> --- any --X--> unsigned
> -fail_format_change(2, 'unsigned')
> -
> --- unsigned -----> any
> -ok_format_change(3, 'any')
> --- unsigned --X--> string
> -fail_format_change(3, 'string')
> --- unsigned -----> number
> -ok_format_change(3, 'number')
> --- unsigned -----> integer
> -ok_format_change(3, 'integer')
> --- unsigned -----> scalar
> -ok_format_change(3, 'scalar')
> --- unsigned --X--> map
> -fail_format_change(3, 'map')
> -
> --- string -----> any
> -ok_format_change(4, 'any')
> --- string -----> scalar
> -ok_format_change(4, 'scalar')
> --- string --X--> boolean
> -fail_format_change(4, 'boolean')
> -
> --- number -----> any
> -ok_format_change(5, 'any')
> --- number -----> scalar
> -ok_format_change(5, 'scalar')
> --- number --X--> integer
> -fail_format_change(5, 'integer')
> -
> --- integer -----> any
> -ok_format_change(6, 'any')
> --- integer -----> number
> -ok_format_change(6, 'number')
> --- integer -----> scalar
> -ok_format_change(6, 'scalar')
> --- integer --X--> unsigned
> -fail_format_change(6, 'unsigned')
> -
> --- boolean -----> any
> -ok_format_change(7, 'any')
> --- boolean -----> scalar
> -ok_format_change(7, 'scalar')
> --- boolean --X--> string
> -fail_format_change(7, 'string')
> -
> --- scalar -----> any
> -ok_format_change(8, 'any')
> --- scalar --X--> unsigned
> -fail_format_change(8, 'unsigned')
> -
> --- array -----> any
> -ok_format_change(9, 'any')
> --- array --X--> scalar
> -fail_format_change(9, 'scalar')
> -
> --- map -----> any
> -ok_format_change(10, 'any')
> --- map --X--> scalar
> -fail_format_change(10, 'scalar')
> -
> -s:drop()
> -
> --- Check new fields adding.
> -format = {}
> -s = box.schema.space.create('test')
> -format[1] = {name = 'field1', type = 'unsigned'}
> -s:format(format) -- Ok, no indexes.
> -pk = s:create_index('pk')
> -format[2] = {name = 'field2', type = 'unsigned'}
> -s:format(format) -- Ok, empty space.
> -s:replace{1, 1}
> -format[2] = nil
> -s:format(format) -- Ok, can delete fields with no checks.
> -s:delete{1}
> -sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
> -sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
> -sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
> -s:replace{1, 1, 1, 1, 1}
> -format[2] = {name = 'field2', type = 'unsigned'}
> -format[3] = {name = 'field3', type = 'unsigned'}
> -format[4] = {name = 'field4', type = 'any'}
> -format[5] = {name = 'field5', type = 'unsigned'}
> --- Ok, all new fields are indexed or have type ANY, and new
> --- field_count <= old field_count.
> -s:format(format)
> -
> -s:replace{1, 1, 1, 1, 1, 1}
> -format[6] = {name = 'field6', type = 'unsigned'}
> --- Ok, but check existing tuples for a new field[6].
> -s:format(format)
> -
> --- Fail, not enough fields.
> -s:replace{2, 2, 2, 2, 2}
> -
> -s:replace{2, 2, 2, 2, 2, 2, 2}
> -format[7] = {name = 'field7', type = 'unsigned'}
> --- Fail, the tuple {1, ... 1} is invalid for a new format.
> -s:format(format)
> -s:drop()
> -
> --- Vinyl does not support adding fields to a not empty space.
> -s = box.schema.space.create('test', {engine = 'vinyl'})
> -pk = s:create_index('pk')
> -s:replace{1,1}
> -format = {}
> -format[1] = {name = 'field1', type = 'unsigned'}
> -format[2] = {name = 'field2', type = 'unsigned'}
> -s:format(format)
> -s:drop()
> -
> ---
>  -- gh-1557: NULL in indexes.
>  --
>  
> @@ -696,34 +405,6 @@ s:select{}
>  s:drop()
>  
>  --
> --- Allow to restrict space format, if corresponding restrictions
> --- already are defined in indexes.
> ---
> -test_run:cmd("setopt delimiter ';'")
> -function check_format_restriction(engine, name)
> -    local s = box.schema.create_space(name, {engine = engine})
> -    local pk = s:create_index('pk')
> -    local format = {}
> -    format[1] = {name = 'field1'}
> -    s:replace{1}
> -    s:replace{100}
> -    s:replace{0}
> -    s:format(format)
> -    s:format()
> -    format[1].type = 'unsigned'
> -    s:format(format)
> -end;
> -test_run:cmd("setopt delimiter ''");
> -check_format_restriction('memtx', 'test1')
> -check_format_restriction('vinyl', 'test2')
> -box.space.test1:format()
> -box.space.test1:select{}
> -box.space.test2:format()
> -box.space.test2:select{}
> -box.space.test1:drop()
> -box.space.test2:drop()
> -
> ---
>  -- Allow to change is_nullable in index definition on non-empty
>  -- space.
>  --
> diff --git a/test/engine/ddl.result b/test/engine/ddl.result
> index 308aefb0..04062ac1 100644
> --- a/test/engine/ddl.result
> +++ b/test/engine/ddl.result
> @@ -357,7 +357,7 @@ space:drop()
>  format = {{'field1', 'scalar'}}
>  ---
>  ...
> -s = box.schema.create_space('test', {format = format})
> +s = box.schema.space.create('test', {engine = engine, format = format})
>  ---
>  ...
>  pk = s:create_index('pk')
> @@ -374,7 +374,7 @@ s:drop()
>  format = {{'field1'}}
>  ---
>  ...
> -s = box.schema.create_space('test', {format = format})
> +s = box.schema.space.create('test', {engine = engine, format = format})
>  ---
>  ...
>  pk = s:create_index('pk')
> @@ -391,7 +391,7 @@ s:drop()
>  -- gh-3229: update optionality if a space format is changed too,
>  -- not only when indexes are updated.
>  --
> -s = box.schema.create_space('test', {engine = engine})
> +s = box.schema.space.create('test', {engine = engine})
>  ---
>  ...
>  format = {}
> @@ -466,7 +466,7 @@ s:drop()
>  --
>  -- Modify key definition without index rebuild.
>  --
> -s = box.schema.create_space('test', {engine = engine})
> +s = box.schema.space.create('test', {engine = engine})
>  ---
>  ...
>  i1 = s:create_index('i1', {unique = true,  parts = {1, 'unsigned'}})
> @@ -573,3 +573,782 @@ i3:select()
>  s:drop()
>  ---
>  ...
> +--
> +-- gh-2652: validate space format.
> +--
> +s = box.schema.space.create('test', { engine = engine, format = "format" })
> +---
> +- error: Illegal parameters, options parameter 'format' should be of type table
> +...
> +format = { { name = 100 } }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +- error: 'Illegal parameters, format[1]: name (string) is expected'
> +...
> +long = string.rep('a', box.schema.NAME_MAX + 1)
> +---
> +...
> +format = { { name = long } }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +- error: 'Failed to create space ''test'': field 1 name is too long'
> +...
> +format = { { name = 'id', type = '100' } }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +- error: 'Failed to create space ''test'': field 1 has unknown field type'
> +...
> +format = { setmetatable({}, { __serialize = 'map' }) }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +- error: 'Illegal parameters, format[1]: name (string) is expected'
> +...
> +-- Ensure the format is updated after index drop.
> +format = { { name = 'id', type = 'unsigned' } }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +sk = s:create_index('sk', { parts = { 2, 'string' } })
> +---
> +...
> +s:replace{1, 1}
> +---
> +- error: 'Tuple field 2 type does not match one required by operation: expected string'
> +...
> +sk:drop()
> +---
> +...
> +s:replace{1, 1}
> +---
> +- [1, 1]
> +...
> +s:drop()
> +---
> +...
> +-- Check index parts conflicting with space format.
> +format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
> +---
> +- error: Field 'field2' has type 'string' in space format, but type 'unsigned' in
> +    index definition
> +...
> +-- Check space format conflicting with index parts.
> +sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
> +---
> +...
> +format[2].type = 'unsigned'
> +---
> +...
> +s:format(format)
> +---
> +- error: Field 'field2' has type 'unsigned' in space format, but type 'string' in
> +    index definition
> +...
> +s:format()
> +---
> +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'string'}, {
> +    'name': 'field3', 'type': 'scalar'}]
> +...
> +s.index.sk3.parts
> +---
> +- - type: string
> +    is_nullable: false
> +    fieldno: 2
> +...
> +-- Space format can be updated, if conflicted index is deleted.
> +sk3:drop()
> +---
> +...
> +s:format(format)
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'unsigned'},
> +  {'name': 'field3', 'type': 'scalar'}]
> +...
> +-- Check deprecated field types.
> +format[2].type = 'num'
> +---
> +...
> +format[3].type = 'str'
> +---
> +...
> +format[4] = { name = 'field4', type = '*' }
> +---
> +...
> +format
> +---
> +- - name: field1
> +    type: unsigned
> +  - name: field2
> +    type: num
> +  - name: field3
> +    type: str
> +  - name: field4
> +    type: '*'
> +...
> +s:format(format)
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'field1', 'type': 'unsigned'}, {'name': 'field2', 'type': 'num'}, {'name': 'field3',
> +    'type': 'str'}, {'name': 'field4', 'type': '*'}]
> +...
> +s:replace{1, 2, '3', {4, 4, 4}}
> +---
> +- [1, 2, '3', [4, 4, 4]]
> +...
> +-- Check not indexed fields checking.
> +s:truncate()
> +---
> +...
> +format[2] = {name='field2', type='string'}
> +---
> +...
> +format[3] = {name='field3', type='array'}
> +---
> +...
> +format[4] = {name='field4', type='number'}
> +---
> +...
> +format[5] = {name='field5', type='integer'}
> +---
> +...
> +format[6] = {name='field6', type='scalar'}
> +---
> +...
> +format[7] = {name='field7', type='map'}
> +---
> +...
> +format[8] = {name='field8', type='any'}
> +---
> +...
> +format[9] = {name='field9'}
> +---
> +...
> +s:format(format)
> +---
> +...
> +-- Check incorrect field types.
> +format[9] = {name='err', type='any'}
> +---
> +...
> +s:format(format)
> +---
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> +---
> +- [1, '2', [3, 3], 4.4, -5, true, {'value': 7}, 8, 9]
> +...
> +s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> +---
> +- error: 'Tuple field 2 type does not match one required by operation: expected string'
> +...
> +s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
> +---
> +- error: 'Tuple field 3 type does not match one required by operation: expected array'
> +...
> +s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
> +---
> +- error: 'Tuple field 4 type does not match one required by operation: expected number'
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
> +---
> +- error: 'Tuple field 5 type does not match one required by operation: expected integer'
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
> +---
> +- error: 'Tuple field 6 type does not match one required by operation: expected scalar'
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
> +---
> +- error: 'Tuple field 7 type does not match one required by operation: expected map'
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
> +---
> +- error: Tuple field count 7 is less than required by space format or defined indexes
> +    (expected at least 9)
> +...
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
> +---
> +- error: Tuple field count 8 is less than required by space format or defined indexes
> +    (expected at least 9)
> +...
> +s:truncate()
> +---
> +...
> +--
> +-- gh-1014: field names.
> +--
> +format = {}
> +---
> +...
> +format[1] = {name = 'field1', type = 'unsigned'}
> +---
> +...
> +format[2] = {name = 'field2'}
> +---
> +...
> +format[3] = {name = 'field1'}
> +---
> +...
> +s:format(format)
> +---
> +- error: Space field 'field1' is duplicate
> +...
> +s:drop()
> +---
> +...
> +-- https://github.com/tarantool/tarantool/issues/2815
> +-- Extend space format definition syntax
> +format = {{name='key',type='unsigned'}, {name='value',type='string'}}
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'key', 'type': 'unsigned'}, {'name': 'value', 'type': 'string'}]
> +...
> +s:format({'id', 'name'})
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'any'}]
> +...
> +s:format({'id', {'name1'}})
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'id', 'type': 'any'}, {'name': 'name1', 'type': 'any'}]
> +...
> +s:format({'id', {'name2', 'string'}})
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'id', 'type': 'any'}, {'name': 'name2', 'type': 'string'}]
> +...
> +s:format({'id', {'name', type = 'string'}})
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'id', 'type': 'any'}, {'name': 'name', 'type': 'string'}]
> +...
> +s:drop()
> +---
> +...
> +format = {'key', {'value',type='string'}}
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'key', 'type': 'any'}, {'name': 'value', 'type': 'string'}]
> +...
> +s:drop()
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine })
> +---
> +...
> +s:create_index('test', {parts = {'test'}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test', {parts = {{'test'}}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test', {parts = {{field = 'test'}}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test', {parts = {1}}).parts
> +---
> +- - type: scalar
> +    is_nullable: false
> +    fieldno: 1
> +...
> +s:drop()
> +---
> +...
> +s = box.schema.space.create('test', { engine = engine })
> +---
> +...
> +s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
> +---
> +...
> +s:create_index('test', {parts = {'test'}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test', {parts = {{'test'}}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test', {parts = {{field = 'test'}}})
> +---
> +- error: 'Illegal parameters, options.parts[1]: field was not found by name ''test'''
> +...
> +s:create_index('test1', {parts = {'test1'}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 1
> +...
> +s:create_index('test2', {parts = {'test2'}}).parts
> +---
> +- error: 'Can''t create or modify index ''test2'' in space ''test'': field type ''any''
> +    is not supported'
> +...
> +s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 1
> +...
> +s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 2
> +...
> +s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 2
> +...
> +s:create_index('test6', {parts = {1, 3}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 1
> +  - type: integer
> +    is_nullable: false
> +    fieldno: 3
> +...
> +s:create_index('test7', {parts = {'test1', 4}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 1
> +  - type: scalar
> +    is_nullable: false
> +    fieldno: 4
> +...
> +s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
> +---
> +- - type: integer
> +    is_nullable: false
> +    fieldno: 1
> +  - type: scalar
> +    is_nullable: false
> +    fieldno: 4
> +...
> +s:drop()
> +---
> +...
> +--
> +-- gh-2800: space formats checking is broken.
> +--
> +-- Ensure that vinyl correctly process field count change.
> +s = box.schema.space.create('test', {engine = engine, field_count = 2})
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +s:replace{1, 2}
> +---
> +- [1, 2]
> +...
> +t = box.space._space:select{s.id}[1]:totable()
> +---
> +...
> +t[5] = 1
> +---
> +...
> +box.space._space:replace(t)
> +---
> +- error: Tuple field count 2 does not match space field count 1
> +...
> +s:drop()
> +---
> +...
> +-- Check field type changes.
> +format = {}
> +---
> +...
> +format[1] = {name = 'field1', type = 'unsigned'}
> +---
> +...
> +format[2] = {name = 'field2', type = 'any'}
> +---
> +...
> +format[3] = {name = 'field3', type = 'unsigned'}
> +---
> +...
> +format[4] = {name = 'field4', type = 'string'}
> +---
> +...
> +format[5] = {name = 'field5', type = 'number'}
> +---
> +...
> +format[6] = {name = 'field6', type = 'integer'}
> +---
> +...
> +format[7] = {name = 'field7', type = 'boolean'}
> +---
> +...
> +format[8] = {name = 'field8', type = 'scalar'}
> +---
> +...
> +format[9] = {name = 'field9', type = 'array'}
> +---
> +...
> +format[10] = {name = 'field10', type = 'map'}
> +---
> +...
> +s = box.schema.space.create('test', {engine = engine, format = format})
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
> +---
> +...
> +inspector:cmd("setopt delimiter ';'")
> +---
> +- true
> +...
> +function fail_format_change(fieldno, new_type)
> +    local old_type = format[fieldno].type
> +    format[fieldno].type = new_type
> +    local ok, msg = pcall(s.format, s, format)
> +    format[fieldno].type = old_type
> +    return msg
> +end;
> +---
> +...
> +function ok_format_change(fieldno, new_type)
> +    local old_type = format[fieldno].type
> +    format[fieldno].type = new_type
> +    s:format(format)
> +    s:delete{1}
> +    format[fieldno].type = old_type
> +    s:format(format)
> +    s:replace(t)
> +end;
> +---
> +...
> +inspector:cmd("setopt delimiter ''");
> +---
> +- true
> +...
> +-- any --X--> unsigned
> +fail_format_change(2, 'unsigned')
> +---
> +- 'Tuple field 2 type does not match one required by operation: expected unsigned'
> +...
> +-- unsigned -----> any
> +ok_format_change(3, 'any')
> +---
> +...
> +-- unsigned --X--> string
> +fail_format_change(3, 'string')
> +---
> +- 'Tuple field 3 type does not match one required by operation: expected string'
> +...
> +-- unsigned -----> number
> +ok_format_change(3, 'number')
> +---
> +...
> +-- unsigned -----> integer
> +ok_format_change(3, 'integer')
> +---
> +...
> +-- unsigned -----> scalar
> +ok_format_change(3, 'scalar')
> +---
> +...
> +-- unsigned --X--> map
> +fail_format_change(3, 'map')
> +---
> +- 'Tuple field 3 type does not match one required by operation: expected map'
> +...
> +-- string -----> any
> +ok_format_change(4, 'any')
> +---
> +...
> +-- string -----> scalar
> +ok_format_change(4, 'scalar')
> +---
> +...
> +-- string --X--> boolean
> +fail_format_change(4, 'boolean')
> +---
> +- 'Tuple field 4 type does not match one required by operation: expected boolean'
> +...
> +-- number -----> any
> +ok_format_change(5, 'any')
> +---
> +...
> +-- number -----> scalar
> +ok_format_change(5, 'scalar')
> +---
> +...
> +-- number --X--> integer
> +fail_format_change(5, 'integer')
> +---
> +- 'Tuple field 5 type does not match one required by operation: expected integer'
> +...
> +-- integer -----> any
> +ok_format_change(6, 'any')
> +---
> +...
> +-- integer -----> number
> +ok_format_change(6, 'number')
> +---
> +...
> +-- integer -----> scalar
> +ok_format_change(6, 'scalar')
> +---
> +...
> +-- integer --X--> unsigned
> +fail_format_change(6, 'unsigned')
> +---
> +- 'Tuple field 6 type does not match one required by operation: expected unsigned'
> +...
> +-- boolean -----> any
> +ok_format_change(7, 'any')
> +---
> +...
> +-- boolean -----> scalar
> +ok_format_change(7, 'scalar')
> +---
> +...
> +-- boolean --X--> string
> +fail_format_change(7, 'string')
> +---
> +- 'Tuple field 7 type does not match one required by operation: expected string'
> +...
> +-- scalar -----> any
> +ok_format_change(8, 'any')
> +---
> +...
> +-- scalar --X--> unsigned
> +fail_format_change(8, 'unsigned')
> +---
> +- 'Tuple field 8 type does not match one required by operation: expected unsigned'
> +...
> +-- array -----> any
> +ok_format_change(9, 'any')
> +---
> +...
> +-- array --X--> scalar
> +fail_format_change(9, 'scalar')
> +---
> +- 'Tuple field 9 type does not match one required by operation: expected scalar'
> +...
> +-- map -----> any
> +ok_format_change(10, 'any')
> +---
> +...
> +-- map --X--> scalar
> +fail_format_change(10, 'scalar')
> +---
> +- 'Tuple field 10 type does not match one required by operation: expected scalar'
> +...
> +s:drop()
> +---
> +...
> +-- Check new fields adding.
> +format = {}
> +---
> +...
> +s = box.schema.space.create('test', {engine = engine})
> +---
> +...
> +format[1] = {name = 'field1', type = 'unsigned'}
> +---
> +...
> +s:format(format) -- Ok, no indexes.
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +format[2] = {name = 'field2', type = 'unsigned'}
> +---
> +...
> +s:format(format) -- Ok, empty space.
> +---
> +...
> +s:replace{1, 1}
> +---
> +- [1, 1]
> +...
> +format[2] = nil
> +---
> +...
> +s:format(format) -- Ok, can delete fields with no checks.
> +---
> +...
> +s:drop()
> +---
> +...
> +s = box.schema.space.create('test', {engine = engine, format = format})
> +---
> +...
> +pk = s:create_index('pk')
> +---
> +...
> +sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
> +---
> +...
> +sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
> +---
> +...
> +sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
> +---
> +...
> +s:replace{1, 1, 1, 1, 1}
> +---
> +- [1, 1, 1, 1, 1]
> +...
> +format[2] = {name = 'field2', type = 'unsigned'}
> +---
> +...
> +format[3] = {name = 'field3', type = 'unsigned'}
> +---
> +...
> +format[4] = {name = 'field4', type = 'any'}
> +---
> +...
> +format[5] = {name = 'field5', type = 'unsigned'}
> +---
> +...
> +-- Ok, all new fields are indexed or have type ANY, and new
> +-- field_count <= old field_count.
> +s:format(format)
> +---
> +...
> +s:replace{1, 1, 1, 1, 1, 1}
> +---
> +- [1, 1, 1, 1, 1, 1]
> +...
> +format[6] = {name = 'field6', type = 'unsigned'}
> +---
> +...
> +-- Ok, but check existing tuples for a new field[6].
> +s:format(format)
> +---
> +...
> +-- Fail, not enough fields.
> +s:replace{2, 2, 2, 2, 2}
> +---
> +- error: Tuple field count 5 is less than required by space format or defined indexes
> +    (expected at least 6)
> +...
> +s:replace{2, 2, 2, 2, 2, 2, 2}
> +---
> +- [2, 2, 2, 2, 2, 2, 2]
> +...
> +format[7] = {name = 'field7', type = 'unsigned'}
> +---
> +...
> +-- Fail, the tuple {1, ... 1} is invalid for a new format.
> +s:format(format)
> +---
> +- error: Tuple field count 6 is less than required by space format or defined indexes
> +    (expected at least 7)
> +...
> +s:drop()
> +---
> +...
> +--
> +-- Allow to restrict space format, if corresponding restrictions
> +-- already are defined in indexes.
> +--
> +s = box.schema.space.create('test', {engine = engine})
> +---
> +...
> +_ = s:create_index('pk')
> +---
> +...
> +format = {}
> +---
> +...
> +format[1] = {name = 'field1'}
> +---
> +...
> +s:replace{1}
> +---
> +- [1]
> +...
> +s:replace{100}
> +---
> +- [100]
> +...
> +s:replace{0}
> +---
> +- [0]
> +...
> +s:format(format)
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'field1', 'type': 'any'}]
> +...
> +format[1].type = 'unsigned'
> +---
> +...
> +s:format(format)
> +---
> +...
> +s:format()
> +---
> +- [{'name': 'field1', 'type': 'unsigned'}]
> +...
> +s:select()
> +---
> +- - [0]
> +  - [1]
> +  - [100]
> +...
> +s:drop()
> +---
> +...
> diff --git a/test/engine/ddl.test.lua b/test/engine/ddl.test.lua
> index 019e18a1..f3d68e1b 100644
> --- a/test/engine/ddl.test.lua
> +++ b/test/engine/ddl.test.lua
> @@ -125,14 +125,14 @@ space:drop()
>  -- is omited.
>  --
>  format = {{'field1', 'scalar'}}
> -s = box.schema.create_space('test', {format = format})
> +s = box.schema.space.create('test', {engine = engine, format = format})
>  pk = s:create_index('pk')
>  pk.parts[1].type
>  s:drop()
>  
>  -- Ensure type 'any' to be not inherited.
>  format = {{'field1'}}
> -s = box.schema.create_space('test', {format = format})
> +s = box.schema.space.create('test', {engine = engine, format = format})
>  pk = s:create_index('pk')
>  pk.parts[1].type
>  s:drop()
> @@ -141,7 +141,7 @@ s:drop()
>  -- gh-3229: update optionality if a space format is changed too,
>  -- not only when indexes are updated.
>  --
> -s = box.schema.create_space('test', {engine = engine})
> +s = box.schema.space.create('test', {engine = engine})
>  format = {}
>  format[1] = {'field1', 'unsigned'}
>  format[2] = {'field2', 'unsigned', is_nullable = true}
> @@ -165,7 +165,7 @@ s:drop()
>  --
>  -- Modify key definition without index rebuild.
>  --
> -s = box.schema.create_space('test', {engine = engine})
> +s = box.schema.space.create('test', {engine = engine})
>  i1 = s:create_index('i1', {unique = true,  parts = {1, 'unsigned'}})
>  i2 = s:create_index('i2', {unique = false, parts = {2, 'unsigned'}})
>  i3 = s:create_index('i3', {unique = true,  parts = {3, 'unsigned'}})
> @@ -195,3 +195,306 @@ i2:select()
>  i3:select()
>  
>  s:drop()
> +
> +--
> +-- gh-2652: validate space format.
> +--
> +s = box.schema.space.create('test', { engine = engine, format = "format" })
> +format = { { name = 100 } }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +long = string.rep('a', box.schema.NAME_MAX + 1)
> +format = { { name = long } }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +format = { { name = 'id', type = '100' } }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +format = { setmetatable({}, { __serialize = 'map' }) }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +
> +-- Ensure the format is updated after index drop.
> +format = { { name = 'id', type = 'unsigned' } }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +pk = s:create_index('pk')
> +sk = s:create_index('sk', { parts = { 2, 'string' } })
> +s:replace{1, 1}
> +sk:drop()
> +s:replace{1, 1}
> +s:drop()
> +
> +-- Check index parts conflicting with space format.
> +format = { { name='field1', type='unsigned' }, { name='field2', type='string' }, { name='field3', type='scalar' } }
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +pk = s:create_index('pk')
> +sk1 = s:create_index('sk1', { parts = { 2, 'unsigned' } })
> +
> +-- Check space format conflicting with index parts.
> +sk3 = s:create_index('sk3', { parts = { 2, 'string' } })
> +format[2].type = 'unsigned'
> +s:format(format)
> +s:format()
> +s.index.sk3.parts
> +
> +-- Space format can be updated, if conflicted index is deleted.
> +sk3:drop()
> +s:format(format)
> +s:format()
> +
> +-- Check deprecated field types.
> +format[2].type = 'num'
> +format[3].type = 'str'
> +format[4] = { name = 'field4', type = '*' }
> +format
> +s:format(format)
> +s:format()
> +s:replace{1, 2, '3', {4, 4, 4}}
> +
> +-- Check not indexed fields checking.
> +s:truncate()
> +format[2] = {name='field2', type='string'}
> +format[3] = {name='field3', type='array'}
> +format[4] = {name='field4', type='number'}
> +format[5] = {name='field5', type='integer'}
> +format[6] = {name='field6', type='scalar'}
> +format[7] = {name='field7', type='map'}
> +format[8] = {name='field8', type='any'}
> +format[9] = {name='field9'}
> +s:format(format)
> +
> +-- Check incorrect field types.
> +format[9] = {name='err', type='any'}
> +s:format(format)
> +
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> +s:replace{1, 2, {3, 3}, 4.4, -5, true, {value=7}, 8, 9}
> +s:replace{1, '2', 3, 4.4, -5, true, {value=7}, 8, 9}
> +s:replace{1, '2', {3, 3}, '4', -5, true, {value=7}, 8, 9}
> +s:replace{1, '2', {3, 3}, 4.4, -5.5, true, {value=7}, 8, 9}
> +s:replace{1, '2', {3, 3}, 4.4, -5, {6, 6}, {value=7}, 8, 9}
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {7}, 8, 9}
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}}
> +s:replace{1, '2', {3, 3}, 4.4, -5, true, {value=7}, 8}
> +s:truncate()
> +
> +--
> +-- gh-1014: field names.
> +--
> +format = {}
> +format[1] = {name = 'field1', type = 'unsigned'}
> +format[2] = {name = 'field2'}
> +format[3] = {name = 'field1'}
> +s:format(format)
> +
> +s:drop()
> +
> +-- https://github.com/tarantool/tarantool/issues/2815
> +-- Extend space format definition syntax
> +format = {{name='key',type='unsigned'}, {name='value',type='string'}}
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +s:format()
> +s:format({'id', 'name'})
> +s:format()
> +s:format({'id', {'name1'}})
> +s:format()
> +s:format({'id', {'name2', 'string'}})
> +s:format()
> +s:format({'id', {'name', type = 'string'}})
> +s:format()
> +s:drop()
> +
> +format = {'key', {'value',type='string'}}
> +s = box.schema.space.create('test', { engine = engine, format = format })
> +s:format()
> +s:drop()
> +
> +s = box.schema.space.create('test', { engine = engine })
> +s:create_index('test', {parts = {'test'}})
> +s:create_index('test', {parts = {{'test'}}})
> +s:create_index('test', {parts = {{field = 'test'}}})
> +s:create_index('test', {parts = {1}}).parts
> +s:drop()
> +
> +s = box.schema.space.create('test', { engine = engine })
> +s:format{{'test1', 'integer'}, 'test2', {'test3', 'integer'}, {'test4','scalar'}}
> +s:create_index('test', {parts = {'test'}})
> +s:create_index('test', {parts = {{'test'}}})
> +s:create_index('test', {parts = {{field = 'test'}}})
> +s:create_index('test1', {parts = {'test1'}}).parts
> +s:create_index('test2', {parts = {'test2'}}).parts
> +s:create_index('test3', {parts = {{'test1', 'integer'}}}).parts
> +s:create_index('test4', {parts = {{'test2', 'integer'}}}).parts
> +s:create_index('test5', {parts = {{'test2', 'integer'}}}).parts
> +s:create_index('test6', {parts = {1, 3}}).parts
> +s:create_index('test7', {parts = {'test1', 4}}).parts
> +s:create_index('test8', {parts = {{1, 'integer'}, {'test4', 'scalar'}}}).parts
> +s:drop()
> +
> +--
> +-- gh-2800: space formats checking is broken.
> +--
> +
> +-- Ensure that vinyl correctly process field count change.
> +s = box.schema.space.create('test', {engine = engine, field_count = 2})
> +pk = s:create_index('pk')
> +s:replace{1, 2}
> +t = box.space._space:select{s.id}[1]:totable()
> +t[5] = 1
> +box.space._space:replace(t)
> +s:drop()
> +
> +-- Check field type changes.
> +format = {}
> +format[1] = {name = 'field1', type = 'unsigned'}
> +format[2] = {name = 'field2', type = 'any'}
> +format[3] = {name = 'field3', type = 'unsigned'}
> +format[4] = {name = 'field4', type = 'string'}
> +format[5] = {name = 'field5', type = 'number'}
> +format[6] = {name = 'field6', type = 'integer'}
> +format[7] = {name = 'field7', type = 'boolean'}
> +format[8] = {name = 'field8', type = 'scalar'}
> +format[9] = {name = 'field9', type = 'array'}
> +format[10] = {name = 'field10', type = 'map'}
> +s = box.schema.space.create('test', {engine = engine, format = format})
> +pk = s:create_index('pk')
> +t = s:replace{1, {2}, 3, '4', 5.5, -6, true, -8, {9, 9}, {val = 10}}
> +
> +inspector:cmd("setopt delimiter ';'")
> +function fail_format_change(fieldno, new_type)
> +    local old_type = format[fieldno].type
> +    format[fieldno].type = new_type
> +    local ok, msg = pcall(s.format, s, format)
> +    format[fieldno].type = old_type
> +    return msg
> +end;
> +
> +function ok_format_change(fieldno, new_type)
> +    local old_type = format[fieldno].type
> +    format[fieldno].type = new_type
> +    s:format(format)
> +    s:delete{1}
> +    format[fieldno].type = old_type
> +    s:format(format)
> +    s:replace(t)
> +end;
> +inspector:cmd("setopt delimiter ''");
> +
> +-- any --X--> unsigned
> +fail_format_change(2, 'unsigned')
> +
> +-- unsigned -----> any
> +ok_format_change(3, 'any')
> +-- unsigned --X--> string
> +fail_format_change(3, 'string')
> +-- unsigned -----> number
> +ok_format_change(3, 'number')
> +-- unsigned -----> integer
> +ok_format_change(3, 'integer')
> +-- unsigned -----> scalar
> +ok_format_change(3, 'scalar')
> +-- unsigned --X--> map
> +fail_format_change(3, 'map')
> +
> +-- string -----> any
> +ok_format_change(4, 'any')
> +-- string -----> scalar
> +ok_format_change(4, 'scalar')
> +-- string --X--> boolean
> +fail_format_change(4, 'boolean')
> +
> +-- number -----> any
> +ok_format_change(5, 'any')
> +-- number -----> scalar
> +ok_format_change(5, 'scalar')
> +-- number --X--> integer
> +fail_format_change(5, 'integer')
> +
> +-- integer -----> any
> +ok_format_change(6, 'any')
> +-- integer -----> number
> +ok_format_change(6, 'number')
> +-- integer -----> scalar
> +ok_format_change(6, 'scalar')
> +-- integer --X--> unsigned
> +fail_format_change(6, 'unsigned')
> +
> +-- boolean -----> any
> +ok_format_change(7, 'any')
> +-- boolean -----> scalar
> +ok_format_change(7, 'scalar')
> +-- boolean --X--> string
> +fail_format_change(7, 'string')
> +
> +-- scalar -----> any
> +ok_format_change(8, 'any')
> +-- scalar --X--> unsigned
> +fail_format_change(8, 'unsigned')
> +
> +-- array -----> any
> +ok_format_change(9, 'any')
> +-- array --X--> scalar
> +fail_format_change(9, 'scalar')
> +
> +-- map -----> any
> +ok_format_change(10, 'any')
> +-- map --X--> scalar
> +fail_format_change(10, 'scalar')
> +
> +s:drop()
> +
> +-- Check new fields adding.
> +format = {}
> +s = box.schema.space.create('test', {engine = engine})
> +format[1] = {name = 'field1', type = 'unsigned'}
> +s:format(format) -- Ok, no indexes.
> +pk = s:create_index('pk')
> +format[2] = {name = 'field2', type = 'unsigned'}
> +s:format(format) -- Ok, empty space.
> +s:replace{1, 1}
> +format[2] = nil
> +s:format(format) -- Ok, can delete fields with no checks.
> +s:drop()
> +
> +s = box.schema.space.create('test', {engine = engine, format = format})
> +pk = s:create_index('pk')
> +sk1 = s:create_index('sk1', {parts = {2, 'unsigned'}})
> +sk2 = s:create_index('sk2', {parts = {3, 'unsigned'}})
> +sk5 = s:create_index('sk5', {parts = {5, 'unsigned'}})
> +s:replace{1, 1, 1, 1, 1}
> +format[2] = {name = 'field2', type = 'unsigned'}
> +format[3] = {name = 'field3', type = 'unsigned'}
> +format[4] = {name = 'field4', type = 'any'}
> +format[5] = {name = 'field5', type = 'unsigned'}
> +-- Ok, all new fields are indexed or have type ANY, and new
> +-- field_count <= old field_count.
> +s:format(format)
> +
> +s:replace{1, 1, 1, 1, 1, 1}
> +format[6] = {name = 'field6', type = 'unsigned'}
> +-- Ok, but check existing tuples for a new field[6].
> +s:format(format)
> +
> +-- Fail, not enough fields.
> +s:replace{2, 2, 2, 2, 2}
> +
> +s:replace{2, 2, 2, 2, 2, 2, 2}
> +format[7] = {name = 'field7', type = 'unsigned'}
> +-- Fail, the tuple {1, ... 1} is invalid for a new format.
> +s:format(format)
> +s:drop()
> +
> +--
> +-- Allow to restrict space format, if corresponding restrictions
> +-- already are defined in indexes.
> +--
> +s = box.schema.space.create('test', {engine = engine})
> +_ = s:create_index('pk')
> +format = {}
> +format[1] = {name = 'field1'}
> +s:replace{1}
> +s:replace{100}
> +s:replace{0}
> +s:format(format)
> +s:format()
> +format[1].type = 'unsigned'
> +s:format(format)
> +s:format()
> +s:select()
> +s:drop()
> diff --git a/test/vinyl/ddl.result b/test/vinyl/ddl.result
> index 5142f0f2..4607a44e 100644
> --- a/test/vinyl/ddl.result
> +++ b/test/vinyl/ddl.result
> @@ -696,89 +696,6 @@ index = space:create_index('test', { type = 'tree', parts = { 2, 'map' }})
>  space:drop()
>  ---
>  ...
> ---
> --- Allow compatible changes of a non-empty vinyl space.
> ---
> -space = box.schema.create_space('test', { engine = 'vinyl' })
> ----
> -...
> -pk = space:create_index('primary')
> ----
> -...
> -space:replace{1}
> ----
> -- [1]
> -...
> -space:replace{2}
> ----
> -- [2]
> -...
> -format = {}
> ----
> -...
> -format[1] = {name = 'field1'}
> ----
> -...
> -format[2] = {name = 'field2', is_nullable = true}
> ----
> -...
> -format[3] = {name = 'field3', is_nullable = true}
> ----
> -...
> -space:format(format)
> ----
> -...
> -t1 = space:replace{3,4,5}
> ----
> -...
> -t2 = space:replace{4,5}
> ----
> -...
> -t1.field1, t1.field2, t1.field3
> ----
> -- 3
> -- 4
> -- 5
> -...
> -t2.field1, t2.field2, t2.field3
> ----
> -- 4
> -- 5
> -- null
> -...
> -t1 = pk:get{1}
> ----
> -...
> -t1.field1, t1.field2, t1.field3
> ----
> -- 1
> -- null
> -- null
> -...
> -box.snapshot()
> ----
> -- ok
> -...
> -t1 = pk:get{2}
> ----
> -...
> -t1.field1, t1.field2, t1.field3
> ----
> -- 2
> -- null
> -- null
> -...
> --- Forbid incompatible change.
> -format[2].is_nullable = false
> ----
> -...
> -space:format(format)
> ----
> -- error: Vinyl does not support changing format of a non-empty space
> -...
> -space:drop()
> ----
> -...
>  -- gh-3019 default index options
>  box.space._space:insert{512, 1, 'test', 'vinyl', 0, setmetatable({}, {__serialize = 'map'}), {}}
>  ---
> diff --git a/test/vinyl/ddl.test.lua b/test/vinyl/ddl.test.lua
> index c4bd36bb..637a331d 100644
> --- a/test/vinyl/ddl.test.lua
> +++ b/test/vinyl/ddl.test.lua
> @@ -260,32 +260,6 @@ index = space:create_index('test', { type = 'tree', parts = { 2, 'array' }})
>  index = space:create_index('test', { type = 'tree', parts = { 2, 'map' }})
>  space:drop()
>  
> ---
> --- Allow compatible changes of a non-empty vinyl space.
> ---
> -space = box.schema.create_space('test', { engine = 'vinyl' })
> -pk = space:create_index('primary')
> -space:replace{1}
> -space:replace{2}
> -format = {}
> -format[1] = {name = 'field1'}
> -format[2] = {name = 'field2', is_nullable = true}
> -format[3] = {name = 'field3', is_nullable = true}
> -space:format(format)
> -t1 = space:replace{3,4,5}
> -t2 = space:replace{4,5}
> -t1.field1, t1.field2, t1.field3
> -t2.field1, t2.field2, t2.field3
> -t1 = pk:get{1}
> -t1.field1, t1.field2, t1.field3
> -box.snapshot()
> -t1 = pk:get{2}
> -t1.field1, t1.field2, t1.field3
> --- Forbid incompatible change.
> -format[2].is_nullable = false
> -space:format(format)
> -space:drop()
> -
>  -- gh-3019 default index options
>  box.space._space:insert{512, 1, 'test', 'vinyl', 0, setmetatable({}, {__serialize = 'map'}), {}}
>  box.space._index:insert{512, 0, 'pk', 'tree', {unique = true}, {{0, 'unsigned'}}}
> diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result
> index 91351086..fd21f7bb 100644
> --- a/test/vinyl/errinj.result
> +++ b/test/vinyl/errinj.result
> @@ -1320,3 +1320,78 @@ ret
>  s:drop()
>  ---
>  ...
> +--
> +-- Check that ALTER is abroted if a tuple inserted during space
> +-- format change does not conform to the new format.
> +--
> +format = {}
> +---
> +...
> +format[1] = {name = 'field1', type = 'unsigned'}
> +---
> +...
> +format[2] = {name = 'field2', type = 'string', is_nullable = true}
> +---
> +...
> +s = box.schema.space.create('test', {engine = 'vinyl', format = format})
> +---
> +...
> +_ = s:create_index('pk', {page_size = 16})
> +---
> +...
> +pad = string.rep('x', 16)
> +---
> +...
> +for i = 101, 200 do s:replace{i, pad} end
> +---
> +...
> +box.snapshot()
> +---
> +- ok
> +...
> +ch = fiber.channel(1)
> +---
> +...
> +test_run:cmd("setopt delimiter ';'")
> +---
> +- true
> +...
> +_ = fiber.create(function()
> +    fiber.sleep(0.01)
> +    for i = 1, 100 do
> +        s:replace{i, box.NULL}
> +    end
> +    ch:put(true)
> +end);
> +---
> +...
> +test_run:cmd("setopt delimiter ''");
> +---
> +- true
> +...
> +errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0.001)
> +---
> +- ok
> +...
> +format[2].is_nullable = false
> +---
> +...
> +s:format(format) -- must fail
> +---
> +- error: 'Tuple field 2 type does not match one required by operation: expected string'
> +...
> +errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0)
> +---
> +- ok
> +...
> +ch:get()
> +---
> +- true
> +...
> +s:count() -- 200
> +---
> +- 200
> +...
> +s:drop()
> +---
> +...
> diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua
> index 9724a69b..64d04c62 100644
> --- a/test/vinyl/errinj.test.lua
> +++ b/test/vinyl/errinj.test.lua
> @@ -513,3 +513,38 @@ errinj.set("ERRINJ_VY_DELAY_PK_LOOKUP", false)
>  while ret == nil do fiber.sleep(0.01) end
>  ret
>  s:drop()
> +
> +--
> +-- Check that ALTER is abroted if a tuple inserted during space
> +-- format change does not conform to the new format.
> +--
> +format = {}
> +format[1] = {name = 'field1', type = 'unsigned'}
> +format[2] = {name = 'field2', type = 'string', is_nullable = true}
> +s = box.schema.space.create('test', {engine = 'vinyl', format = format})
> +_ = s:create_index('pk', {page_size = 16})
> +
> +pad = string.rep('x', 16)
> +for i = 101, 200 do s:replace{i, pad} end
> +box.snapshot()
> +
> +ch = fiber.channel(1)
> +test_run:cmd("setopt delimiter ';'")
> +_ = fiber.create(function()
> +    fiber.sleep(0.01)
> +    for i = 1, 100 do
> +        s:replace{i, box.NULL}
> +    end
> +    ch:put(true)
> +end);
> +test_run:cmd("setopt delimiter ''");
> +
> +errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0.001)
> +format[2].is_nullable = false
> +s:format(format) -- must fail
> +errinj.set("ERRINJ_VY_READ_PAGE_TIMEOUT", 0)
> +
> +ch:get()
> +
> +s:count() -- 200
> +s:drop()
> -- 
> 2.11.0
> 

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov