Re: [Tarantool-patches] [PATCH vshard 08/11] storage: introduce bucket_are_all_rw()

[Oleg Babin via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

Thanks for your patch.

Seems here I should return to one of my previous e-mail.

Maybe it's reasonable to cache all bucket stats?


On 23.02.2021 03:15, Vladislav Shpilevoy wrote:
> In the future map-reduce code it will be needed to be able to
> check if all buckets on the storage are in writable state. If they
> are - any request can do anything with all the data on the
> storage.
>
> Such 'all writable' state will be pinned by a new module
> 'storage_ref' so as map-reduce requests could execute without
> being afraid of the rebalancer.
>
> The patch adds a function bucket_are_all_rw() which is registered
> in registry.storage.
>
> The function is not trivial because tries to cache the returned
> value. It makes a lot of sense, because the value changes super
> rare and the calculation costs a lot (4 lookups in an index by a
> string key via FFI + each lookup returns a tuple which is +1 Lua
> GC object).
>
> The function is going to be used almost on each map-reduce
> request, so it must be fast.
>
> Part of #147
> ---
>   test/storage/storage.result   | 37 +++++++++++++++++++++++++++++++++++
>   test/storage/storage.test.lua | 14 +++++++++++++
>   vshard/storage/init.lua       | 37 +++++++++++++++++++++++++++++++++++
>   3 files changed, 88 insertions(+)
>
> diff --git a/test/storage/storage.result b/test/storage/storage.result
> index 4730e20..2c9784a 100644
> --- a/test/storage/storage.result
> +++ b/test/storage/storage.result
> @@ -808,6 +808,43 @@ assert(not ok and err.message)
>   ---
>   - fiber is cancelled
>   ...
> +--
> +-- Bucket_are_all_rw() registry function.
> +--
> +assert(lstorage.bucket_are_all_rw())
> +---
> +- true
> +...
> +vshard.storage.internal.errinj.ERRINJ_NO_RECOVERY = true
> +---
> +...
> +-- Let it stuck in the errinj.
> +vshard.storage.recovery_wakeup()
> +---
> +...
> +vshard.storage.bucket_force_create(10)
> +---
> +- true
> +...
> +box.space._bucket:update(10, {{'=', 2, vshard.consts.BUCKET.SENDING}})
> +---
> +- [10, 'sending']
> +...
> +assert(not lstorage.bucket_are_all_rw())
> +---
> +- true
> +...
> +box.space._bucket:update(10, {{'=', 2, vshard.consts.BUCKET.ACTIVE}})
> +---
> +- [10, 'active']
> +...
> +assert(lstorage.bucket_are_all_rw())
> +---
> +- true
> +...
> +vshard.storage.internal.errinj.ERRINJ_NO_RECOVERY = false
> +---
> +...
>   _ = test_run:switch("default") --- ... diff --git a/test/storage/storage.test.lua 
> b/test/storage/storage.test.lua index 86c5e33..33f0498 100644 --- 
> a/test/storage/storage.test.lua +++ b/test/storage/storage.test.lua @@ 
> -241,6 +241,20 @@ f:cancel() _ = test_run:wait_cond(function() return 
> ok or err end) assert(not ok and err.message) +-- +-- 
> Bucket_are_all_rw() registry function. +-- 
> +assert(lstorage.bucket_are_all_rw()) 
> +vshard.storage.internal.errinj.ERRINJ_NO_RECOVERY = true +-- Let it 
> stuck in the errinj. +vshard.storage.recovery_wakeup() 
> +vshard.storage.bucket_force_create(10) +box.space._bucket:update(10, 
> {{'=', 2, vshard.consts.BUCKET.SENDING}}) +assert(not 
> lstorage.bucket_are_all_rw()) +box.space._bucket:update(10, {{'=', 2, 
> vshard.consts.BUCKET.ACTIVE}}) +assert(lstorage.bucket_are_all_rw()) 
> +vshard.storage.internal.errinj.ERRINJ_NO_RECOVERY = false + _ = 
> test_run:switch("default")
>   test_run:drop_cluster(REPLICASET_2)
>   test_run:drop_cluster(REPLICASET_1)
> diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua
> index ffa48b6..c3ed236 100644
> --- a/vshard/storage/init.lua
> +++ b/vshard/storage/init.lua
> @@ -115,6 +115,10 @@ if not M then
>           -- Fast alternative to box.space._bucket:count(). But may be nil. Reset
>           -- on each generation change.
>           bucket_count_cache = nil,
> +        -- Fast alternative to checking multiple keys presence in
> +        -- box.space._bucket status index. But may be nil. Reset on each
> +        -- generation change.
> +        bucket_are_all_rw_cache = nil,
>           -- Redirects for recently sent buckets. They are kept for a while to
>           -- help routers to find a new location for sent and deleted buckets
>           -- without whole cluster scan.
> @@ -220,12 +224,44 @@ local function bucket_count_public()
>       return bucket_count()
>   end
>   
> +--
> +-- Check if all buckets on the storage are writable. The idea is the same as
> +-- with bucket count - the value changes very rare, and is cached most of the
> +-- time. Only that its non-cached calculation is more expensive than with count.
> +--
> +local bucket_are_all_rw
> +
> +local function bucket_are_all_rw_cache()
> +    return M.bucket_are_all_rw_cache
> +end
> +
> +local function bucket_are_all_rw_not_cache()
> +    local status_index = box.space._bucket.index.status
> +    local status = consts.BUCKET
> +    local res = not status_index:min(status.SENDING) and
> +       not status_index:min(status.SENT) and
> +       not status_index:min(status.RECEIVING) and
> +       not status_index:min(status.GARBAGE)
> +
> +    M.bucket_are_all_rw_cache = res
> +    bucket_are_all_rw = bucket_are_all_rw_cache
> +    return res
> +end
> +
> +bucket_are_all_rw = bucket_are_all_rw_not_cache
> +
> +local function bucket_are_all_rw_public()
> +    return bucket_are_all_rw()
> +end
> +
>   --
>   -- Trigger for on replace into _bucket to update its generation.
>   --
>   local function bucket_generation_increment()
>       bucket_count = bucket_count_not_cache
> +    bucket_are_all_rw = bucket_are_all_rw_not_cache
>       M.bucket_count_cache = nil
> +    M.bucket_are_all_rw_cache = nil
>       M.bucket_generation = M.bucket_generation + 1
>       M.bucket_generation_cond:broadcast()
>   end
> @@ -2788,6 +2824,7 @@ M.schema_upgrade_handlers = schema_upgrade_handlers
>   M.schema_version_make = schema_version_make
>   M.schema_bootstrap = schema_init_0_1_15_0
>   
> +M.bucket_are_all_rw = bucket_are_all_rw_public
>   M.bucket_generation_wait = bucket_generation_wait
>   lregistry.storage = M
>