Re: [Tarantool-patches] [PATCH v4 06/16] sql: introduce mem_append()

[Mergen Imeev via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

Hi! Thank you for the review! My answer, diff and new patch below.

On Mon, Oct 11, 2021 at 11:50:08PM +0200, Vladislav Shpilevoy wrote:
> Hi! Thanks for the fixes!
> 
> >>> diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c
> >>> index 079083fa1..b8ead8f41 100644
> >>> --- a/src/box/sql/mem.c
> >>> +++ b/src/box/sql/mem.c
> >>> @@ -1925,6 +1925,27 @@ mem_move(struct Mem *to, struct Mem *from)
> >>>  	from->zMalloc = NULL;
> >>>  }
> >>>  
> >>> +int
> >>> +mem_append(struct Mem *mem, const char *value, uint32_t len)
> >>> +{
> >>> +	assert((mem->type & (MEM_TYPE_BIN | MEM_TYPE_STR)) != 0);
> >>> +	if (len == 0)
> >>> +		return 0;
> >>> +	int new_size = mem->n + len;
> >>> +	if (((mem->flags & (MEM_Static | MEM_Dyn | MEM_Ephem)) != 0) ||
> >>> +	    mem->szMalloc < new_size) {
> >>> +		/*
> >>> +		 * Force exponential buffer size growth to avoid having to call
> >>> +		 * this routine too often.
> >>> +		 */
> >>> +		if (sqlVdbeMemGrow(mem, new_size + mem->n, 1) != 0)
> >>
> >> Looks like you could call sqlVdbeMemGrow() without all these checks
> >> above. The grow function does them already.
> >>
> > True, fixed. There is a case where the behavior will be different - I
> > mean the case where new_size < szMalloc and new_size + n > szMalloc,
> > but that doesn't seem to be a very common case.
> 
> Wait, this does not look correct either. I thought sqlVdbeMemGrow()
> does the exponential grow. But if it does not, we need to either patch
> it, or keep some checks above. The 'standard' way is to grow only when
> necessary, but at least x2. How this new way works I can't tell. It
> might be worse. You will always have unused free space. Lets either
> stick to the old way or patch the grow function.
I think it's better to use the old way, because in all other cases we only need
to reallocate the memory once, and it would be in vain to double it every time.


Diff:

diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c
index 343a730b3..de4c41ccf 100644
--- a/src/box/sql/mem.c
+++ b/src/box/sql/mem.c
@@ -1934,12 +1934,15 @@ mem_append(struct Mem *mem, const char *value, uint32_t len)
 	if (len == 0)
 		return 0;
 	int new_size = mem->n + len;
-	/*
-	 * Force exponential buffer size growth to avoid having to call this
-	 * routine too often.
-	 */
-	if (sqlVdbeMemGrow(mem, new_size + mem->n, 1) != 0)
-		return -1;
+	if (((mem->flags & (MEM_Static | MEM_Dyn | MEM_Ephem)) != 0) ||
+	    mem->szMalloc < new_size) {
+		/*
+		 * Force exponential buffer size growth to avoid having to call
+		 * this routine too often.
+		 */
+		if (sqlVdbeMemGrow(mem, new_size + mem->n, 1) != 0)
+			return -1;
+	}
 	memcpy(&mem->z[mem->n], value, len);
 	mem->n = new_size;
 	return 0;


New patch:

commit 7cb6f95fcf4a85b36e8c4a3f41cbcc5e8741e417
Author: Mergen Imeev <imeevma@gmail.com>
Date:   Mon Sep 6 13:49:47 2021 +0300

    sql: introduce mem_append()
    
    This patch introduces the mem_append() function. This function appends
    the specified string to the end of the STRING or VARBINARY contained in
    MEM. In case MEM needs to increase the size of allocated memory,
    extra memory is allocated in an attempt to reduce the total number of
    allocations.
    
    Needed for #4145

diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c
index f1b61dec6..de4c41ccf 100644
--- a/src/box/sql/mem.c
+++ b/src/box/sql/mem.c
@@ -1927,6 +1927,27 @@ mem_move(struct Mem *to, struct Mem *from)
 	from->zMalloc = NULL;
 }
 
+int
+mem_append(struct Mem *mem, const char *value, uint32_t len)
+{
+	assert((mem->type & (MEM_TYPE_BIN | MEM_TYPE_STR)) != 0);
+	if (len == 0)
+		return 0;
+	int new_size = mem->n + len;
+	if (((mem->flags & (MEM_Static | MEM_Dyn | MEM_Ephem)) != 0) ||
+	    mem->szMalloc < new_size) {
+		/*
+		 * Force exponential buffer size growth to avoid having to call
+		 * this routine too often.
+		 */
+		if (sqlVdbeMemGrow(mem, new_size + mem->n, 1) != 0)
+			return -1;
+	}
+	memcpy(&mem->z[mem->n], value, len);
+	mem->n = new_size;
+	return 0;
+}
+
 int
 mem_concat(const struct Mem *a, const struct Mem *b, struct Mem *result)
 {
diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h
index 1574da22d..b567aea50 100644
--- a/src/box/sql/mem.h
+++ b/src/box/sql/mem.h
@@ -582,6 +582,14 @@ mem_copy_as_ephemeral(struct Mem *to, const struct Mem *from);
 void
 mem_move(struct Mem *to, struct Mem *from);
 
+/**
+ * Append the given string to the end of the STRING or VARBINARY contained in
+ * MEM. In case MEM needs to increase the size of allocated memory, additional
+ * memory is allocated in an attempt to reduce the total number of allocations.
+ */
+int
+mem_append(struct Mem *mem, const char *value, uint32_t len);
+
 /**
  * Concatenate strings or binaries from the first and the second MEMs and write
  * to the result MEM. In case the first MEM or the second MEM is NULL, the
diff --git a/src/box/sql/vdbeaux.c b/src/box/sql/vdbeaux.c
index 4c2bd11ba..3015760e1 100644
--- a/src/box/sql/vdbeaux.c
+++ b/src/box/sql/vdbeaux.c
@@ -1294,14 +1294,9 @@ sqlVdbeList(Vdbe * p)
 					if (mem_copy_bin(pSub, bin, size) != 0)
 						return -1;
 				} else if (j == nSub) {
-					struct Mem tmp;
-					mem_create(&tmp);
-					uint32_t size = sizeof(SubProgram *);
 					char *bin = (char *)&pOp->p4.pProgram;
-					mem_set_bin_ephemeral(&tmp, bin, size);
-					int rc = mem_concat(pSub, &tmp, pSub);
-					mem_destroy(&tmp);
-					if (rc != 0)
+					uint32_t size = sizeof(SubProgram *);
+					if (mem_append(pSub, bin, size) != 0)
 						return -1;
 				}
 			}