Re: [Tarantool-patches] [PATCH v4 10/16] sql: refactor AVG() function

[Mergen Imeev via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

Thank you for the review! My answer, diff and new patch below.

On Mon, Oct 11, 2021 at 11:50:39PM +0200, Vladislav Shpilevoy wrote:
> Thanks for the fixes!
> 
> >> @@ -141,17 +141,14 @@ fin_avg(struct sql_context *ctx)
> >>  	assert(mem_is_null(ctx->pMem) || mem_is_bin(ctx->pMem));
> >>  	if (mem_is_null(ctx->pMem))
> >>  		return mem_set_null(ctx->pOut);
> >> -	struct Mem *tmp = (struct Mem *)ctx->pMem->z;
> >> -	uint32_t *count_val = (uint32_t *)(tmp + 1);
> >> -	struct Mem sum;
> >> -	mem_create(&sum);
> >> -	mem_copy_as_ephemeral(&sum, tmp);
> >> -	mem_destroy(tmp);
> >> +	struct Mem *sum = (struct Mem *)ctx->pMem->z;
> >> +	uint32_t *count_val = (uint32_t *)(sum + 1);
> >>  	struct Mem count;
> >>  	mem_create(&count);
> >>  	mem_set_uint(&count, *count_val);
> >>  	if (mem_div(&sum, &count, ctx->pOut) != 0)
> >>  		ctx->is_aborted = true;
> >> +	mem_destroy(sum);
> >>  }
> > This will work, however, I think it will create some unnecessary restrictions
> > due to changes with pMem and pOut in a few patches. I suggest to apply part of
> > you diff with exception of mem_destroy(), which I sugget to replace by assert().
> > We have full control over this tmp/sum mem and we know, that there will be no
> > memory to free, so assert should be enough.
> > 
> > What do you think of this diff?
> 
> It looks the same as mine except you didn't call the destroy. I am fine with it,
> but when I propose to wrap the check about a mem not needing a destroy into a
> function. We should not use mem members as is when possible. It is a too
> complicated structure. So far.
> 
> Something like mem_is_trivial(). If it returns true, you don't need to
> call mem_clear()/mem_destroy() and nothing will leak.
Added mem_is_trivial().


Diff:

diff --git a/src/box/sql/func.c b/src/box/sql/func.c
index a811e55f9..8be553110 100644
--- a/src/box/sql/func.c
+++ b/src/box/sql/func.c
@@ -141,16 +141,13 @@ fin_avg(struct sql_context *ctx)
 	assert(mem_is_null(ctx->pMem) || mem_is_bin(ctx->pMem));
 	if (mem_is_null(ctx->pMem))
 		return mem_set_null(ctx->pOut);
-	struct Mem *tmp = (struct Mem *)ctx->pMem->z;
-	uint32_t *count_val = (uint32_t *)(tmp + 1);
-	struct Mem sum;
-	mem_create(&sum);
-	mem_copy_as_ephemeral(&sum, tmp);
-	mem_destroy(tmp);
+	struct Mem *sum = (struct Mem *)ctx->pMem->z;
+	uint32_t *count_val = (uint32_t *)(sum + 1);
+	assert(mem_is_trivial(sum));
 	struct Mem count;
 	mem_create(&count);
 	mem_set_uint(&count, *count_val);
-	if (mem_div(&sum, &count, ctx->pOut) != 0)
+	if (mem_div(sum, &count, ctx->pOut) != 0)
 		ctx->is_aborted = true;
 }
 
diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h
index 7d5a750f5..52a63949a 100644
--- a/src/box/sql/mem.h
+++ b/src/box/sql/mem.h
@@ -237,6 +237,14 @@ mem_is_allocated(const struct Mem *mem)
 	return mem_is_bytes(mem) && mem->z == mem->zMalloc;
 }
 
+/** Return TRUE if MEM does not need to be freed or destroyed. */
+static inline bool
+mem_is_trivial(const struct Mem *mem)
+{
+	return mem->szMalloc == 0 && (mem->flags & MEM_Dyn) == 0 &&
+	       (mem->type & (MEM_TYPE_FRAME | MEM_TYPE_AGG)) == 0;
+}
+
 static inline bool
 mem_is_cleared(const struct Mem *mem)
 {


New patch:

commit 18c50ab95a05c958cf1be016a482aa89f121f9b6
Author: Mergen Imeev <imeevma@gmail.com>
Date:   Thu Sep 9 18:19:53 2021 +0300

    sql: refactor AVG() function
    
    Part of #4145

diff --git a/src/box/sql/func.c b/src/box/sql/func.c
index c3c7ebec0..8be553110 100644
--- a/src/box/sql/func.c
+++ b/src/box/sql/func.c
@@ -102,6 +102,55 @@ fin_total(struct sql_context *ctx)
 		mem_copy_as_ephemeral(ctx->pOut, ctx->pMem);
 }
 
+/** Implementation of the AVG() function. */
+static void
+step_avg(struct sql_context *ctx, int argc, struct Mem **argv)
+{
+	assert(argc == 1);
+	(void)argc;
+	assert(mem_is_null(ctx->pMem) || mem_is_bin(ctx->pMem));
+	if (mem_is_null(argv[0]))
+		return;
+	struct Mem *mem;
+	uint32_t *count;
+	if (mem_is_null(ctx->pMem)) {
+		uint32_t size = sizeof(struct Mem) + sizeof(uint32_t);
+		mem = sqlDbMallocRawNN(sql_get(), size);
+		if (mem == NULL) {
+			ctx->is_aborted = true;
+			return;
+		}
+		count = (uint32_t *)(mem + 1);
+		mem_create(mem);
+		*count = 1;
+		mem_copy_as_ephemeral(mem, argv[0]);
+		mem_set_bin_allocated(ctx->pMem, (char *)mem, size);
+		return;
+	}
+	mem = (struct Mem *)ctx->pMem->z;
+	count = (uint32_t *)(mem + 1);
+	++*count;
+	if (mem_add(mem, argv[0], mem) != 0)
+		ctx->is_aborted = true;
+}
+
+/** Finalizer for the AVG() function. */
+static void
+fin_avg(struct sql_context *ctx)
+{
+	assert(mem_is_null(ctx->pMem) || mem_is_bin(ctx->pMem));
+	if (mem_is_null(ctx->pMem))
+		return mem_set_null(ctx->pOut);
+	struct Mem *sum = (struct Mem *)ctx->pMem->z;
+	uint32_t *count_val = (uint32_t *)(sum + 1);
+	assert(mem_is_trivial(sum));
+	struct Mem count;
+	mem_create(&count);
+	mem_set_uint(&count, *count_val);
+	if (mem_div(sum, &count, ctx->pOut) != 0)
+		ctx->is_aborted = true;
+}
+
 static const unsigned char *
 mem_as_ustr(struct Mem *mem)
 {
@@ -1663,69 +1712,6 @@ soundexFunc(sql_context * context, int argc, sql_value ** argv)
 	}
 }
 
-/*
- * An instance of the following structure holds the context of a
- * sum() or avg() aggregate computation.
- */
-typedef struct SumCtx SumCtx;
-struct SumCtx {
-	struct Mem mem;
-	uint32_t count;
-};
-
-/*
- * Routines used to compute the sum, average, and total.
- *
- * The SUM() function follows the (broken) SQL standard which means
- * that it returns NULL if it sums over no inputs.  TOTAL returns
- * 0.0 in that case.  In addition, TOTAL always returns a float where
- * SUM might return an integer if it never encounters a floating point
- * value.  TOTAL never fails, but SUM might through an exception if
- * it overflows an integer.
- */
-static void
-sum_step(struct sql_context *context, int argc, sql_value **argv)
-{
-	assert(argc == 1);
-	UNUSED_PARAMETER(argc);
-	struct SumCtx *p = sql_aggregate_context(context, sizeof(*p));
-	if (p == NULL) {
-		context->is_aborted = true;
-		return;
-	}
-	if (p->count == 0) {
-		mem_create(&p->mem);
-		assert(context->func->def->returns == FIELD_TYPE_INTEGER ||
-		       context->func->def->returns == FIELD_TYPE_DOUBLE);
-		if (context->func->def->returns == FIELD_TYPE_INTEGER)
-			mem_set_uint(&p->mem, 0);
-		else
-			mem_set_double(&p->mem, 0.0);
-	}
-	if (argv[0]->type == MEM_TYPE_NULL)
-		return;
-	++p->count;
-	assert(mem_is_num(argv[0]));
-	if (mem_add(&p->mem, argv[0], &p->mem) != 0)
-		context->is_aborted = true;
-}
-
-static void
-avgFinalize(sql_context * context)
-{
-	SumCtx *p;
-	p = sql_aggregate_context(context, 0);
-	if (p == NULL || p->count == 0) {
-		mem_set_null(context->pOut);
-		return;
-	}
-	struct Mem mem;
-	mem_create(&mem);
-	mem_set_uint(&mem, p->count);
-	if (mem_div(&p->mem, &mem, context->pOut) != 0)
-		context->is_aborted = true;
-}
-
 /*
  * The following structure keeps track of state information for the
  * count() aggregate function.
@@ -2022,8 +2008,8 @@ struct sql_func_definition {
 static struct sql_func_definition definitions[] = {
 	{"ABS", 1, {FIELD_TYPE_INTEGER}, FIELD_TYPE_INTEGER, absFunc, NULL},
 	{"ABS", 1, {FIELD_TYPE_DOUBLE}, FIELD_TYPE_DOUBLE, absFunc, NULL},
-	{"AVG", 1, {FIELD_TYPE_INTEGER}, FIELD_TYPE_INTEGER, sum_step, avgFinalize},
-	{"AVG", 1, {FIELD_TYPE_DOUBLE}, FIELD_TYPE_DOUBLE, sum_step, avgFinalize},
+	{"AVG", 1, {FIELD_TYPE_INTEGER}, FIELD_TYPE_INTEGER, step_avg, fin_avg},
+	{"AVG", 1, {FIELD_TYPE_DOUBLE}, FIELD_TYPE_DOUBLE, step_avg, fin_avg},
 	{"CHAR", -1, {FIELD_TYPE_INTEGER}, FIELD_TYPE_STRING, charFunc, NULL},
 	{"CHAR_LENGTH", 1, {FIELD_TYPE_STRING}, FIELD_TYPE_INTEGER, lengthFunc,
 	 NULL},
diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h
index 7d5a750f5..52a63949a 100644
--- a/src/box/sql/mem.h
+++ b/src/box/sql/mem.h
@@ -237,6 +237,14 @@ mem_is_allocated(const struct Mem *mem)
 	return mem_is_bytes(mem) && mem->z == mem->zMalloc;
 }
 
+/** Return TRUE if MEM does not need to be freed or destroyed. */
+static inline bool
+mem_is_trivial(const struct Mem *mem)
+{
+	return mem->szMalloc == 0 && (mem->flags & MEM_Dyn) == 0 &&
+	       (mem->type & (MEM_TYPE_FRAME | MEM_TYPE_AGG)) == 0;
+}
+
 static inline bool
 mem_is_cleared(const struct Mem *mem)
 {