From 2fde647fc1bd879c41d97513ccd48e3737c6a2e8 Mon Sep 17 00:00:00 2001 From: kostas Date: Thu, 28 Nov 2024 18:51:13 +0200 Subject: [PATCH 01/15] feat: add support for big values in SeederV2 Signed-off-by: kostas --- tests/dragonfly/seeder/__init__.py | 8 ++ tests/dragonfly/seeder/script-generate.lua | 6 +- tests/dragonfly/seeder/script-genlib.lua | 95 +++++++++++++++++++--- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index aeea29c401dd..29e12cb1a0db 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -19,6 +19,7 @@ class SeederBase: UID_COUNTER = 1 # multiple generators should not conflict on keys CACHED_SCRIPTS = {} DEFAULT_TYPES = ["STRING", "LIST", "SET", "HASH", "ZSET", "JSON"] + BIG_VALUE_TYPES = ["LIST", "SET", "HASH", "ZSET"] def __init__(self, types: typing.Optional[typing.List[str]] = None): self.uid = SeederBase.UID_COUNTER @@ -137,6 +138,8 @@ def __init__( data_size=100, collection_size=None, types: typing.Optional[typing.List[str]] = None, + huge_value_percentage=0, + huge_value_size=0, ): SeederBase.__init__(self, types) self.key_target = key_target @@ -146,6 +149,9 @@ def __init__( else: self.collection_size = collection_size + self.huge_value_percentage = huge_value_percentage + self.huge_value_size = huge_value_size + self.units = [ Seeder.Unit( prefix=f"k-s{self.uid}u{i}-", @@ -166,6 +172,8 @@ async def run(self, client: aioredis.Redis, target_ops=None, target_deviation=No target_deviation if target_deviation is not None else -1, self.data_size, self.collection_size, + self.huge_value_percentage, + self.huge_value_size, ] sha = await client.script_load(Seeder._load_script("generate")) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index d1f8184259c5..9cb7489a6073 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -18,12 +18,15 @@ local total_ops = tonumber(ARGV[6]) local min_dev = tonumber(ARGV[7]) local data_size = tonumber(ARGV[8]) local collection_size = tonumber(ARGV[9]) +-- Probability of each key in key_target to be a big value +local huge_value_percentage = tonumber(ARGV[10]) +local huge_value_size = tonumber(ARGV[11]) -- collect all keys belonging to this script -- assumes exclusive ownership local keys = LU_collect_keys(prefix, type) -LG_funcs.init(data_size, collection_size) +LG_funcs.init(data_size, collection_size, huge_value_percentage, huge_value_size) local addfunc = LG_funcs['add_' .. string.lower(type)] local modfunc = LG_funcs['mod_' .. string.lower(type)] @@ -85,6 +88,7 @@ while true do if counter % 10 == 0 then -- calculate intensity (not normalized probabilities) -- please see attached plots in PR to undertand convergence + -- https://github.com/dragonflydb/dragonfly/pull/2556 -- the add intensity is monotonically decreasing with keycount growing, -- the delete intensity is monotonically increasing with keycount growing, diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index 937f26fe4d57..06c4df532e4d 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -1,9 +1,19 @@ local LG_funcs = {} -function LG_funcs.init(dsize, csize) +function LG_funcs.init(dsize, csize, large_val_perc, large_val_sz) LG_funcs.dsize = dsize LG_funcs.csize = csize LG_funcs.esize = math.ceil(dsize / csize) + LG_funcs.huge_value_percentage = large_val_perc + LG_funcs.huge_value_size = large_val_sz +end + +local function huge_entry() + local perc = LG_funcs.huge_value_percentage / 100 + -- [0, 1] + local rand = math.random() + local huge_entry = (perc > rand) + return huge_entry end -- strings @@ -27,12 +37,18 @@ end -- lists -- store list of random blobs of default container/element sizes -function LG_funcs.add_list(key) - local elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) +function LG_funcs.add_list(key, huge_value) + local elements + if huge_entry() then + elements = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) + else + elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) + end + redis.apcall('LPUSH', key, unpack(elements)) end -function LG_funcs.mod_list(key) +function LG_funcs.mod_list(key, huge_value) -- equally likely pops and pushes, we rely on the list size being large enough -- to "highly likely" not get emptied out by consequitve pops local action = math.random(1, 4) @@ -41,9 +57,23 @@ function LG_funcs.mod_list(key) elseif action == 2 then redis.apcall('LPOP', key) elseif action == 3 then - redis.apcall('LPUSH', key, dragonfly.randstr(LG_funcs.esize)) + local str + if huge_entry() then + str = dragonfly.randstr(LG_funcs.huge_value_size) + else + str = dragonfly.randstr(LG_funcs.esize) + end + + redis.apcall('LPUSH', key, str) else - redis.apcall('RPUSH', key, dragonfly.randstr(LG_funcs.esize)) + local str + if huge_entry() then + str = dragonfly.randstr(LG_funcs.huge_value_size) + else + str = dragonfly.randstr(LG_funcs.esize) + end + + redis.apcall('RPUSH', key, str) end end @@ -62,7 +92,15 @@ function LG_funcs.add_set(key, keys) end redis.apcall('SDIFFSTORE', key, keys[i1], keys[i2]) else - local elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) + local elements + if huge_entry() then + -- Hard coded 10 here, meaning up to 10 huge entries per set + -- TODO make this configurable + elements = dragonfly.randstr(LG_funcs.large_val_sz, 10) + else + elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) + end + redis.apcall('SADD', key, unpack(elements)) end end @@ -72,7 +110,14 @@ function LG_funcs.mod_set(key) if math.random() < 0.5 then redis.apcall('SPOP', key) else - redis.apcall('SADD', key, dragonfly.randstr(LG_funcs.esize)) + local rand_str + if huge_entry() then + rand_str = dragonfly.randstr(LG_funcs.huge_value_size) + else + rand_str = dragonfly.randstr(LG_funcs.esize) + end + + redis.apcall('SADD', key, rand_str) end end @@ -82,7 +127,13 @@ end -- where `value` is a random string for even indices and a number for odd indices function LG_funcs.add_hash(key) - local blobs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize / 2) + local blobs + if huge_entry() then + blobs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize / 2) + else + blobs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize / 2) + end + local htable = {} for i = 1, LG_funcs.csize, 2 do htable[i * 2 - 1] = tostring(i) @@ -100,7 +151,14 @@ function LG_funcs.mod_hash(key) if idx % 2 == 1 then redis.apcall('HINCRBY', key, tostring(idx), 1) else - redis.apcall('HSET', key, tostring(idx), dragonfly.randstr(LG_funcs.esize)) + local str + if huge_entry() then + str = dragonfly.randstr(LG_funcs.large_val_sz) + else + str = dragonfly.randstr(LG_funcs.esize) + end + + redis.apcall('HSET', key, tostring(idx), str) end end @@ -108,7 +166,13 @@ end function LG_funcs.add_zset(key, keys) -- TODO: We don't support ZDIFFSTORE - local blobs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) + local blobs + if huge_entry() then + blobs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) + else + blobs = dragonfly.randstr(LG_funcs.csize, LG_funcs.csize) + end + local ztable = {} for i = 1, LG_funcs.csize do ztable[i * 2 - 1] = tostring(i) @@ -120,7 +184,14 @@ end function LG_funcs.mod_zset(key, dbsize) local action = math.random(1, 4) if action <= 2 then - redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), dragonfly.randstr(LG_funcs.esize)) + local str + if huge_entry() then + str = dragonfly.randstr(LG_funcs.large_val_sz) + else + str = dragonfly.randstr(LG_funcs.esize) + end + + redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), str) elseif action == 3 then redis.apcall('ZPOPMAX', key) else From 1f9ccd11f23c19ea6544bdab1dfbb46ef7418e9e Mon Sep 17 00:00:00 2001 From: kostas Date: Fri, 29 Nov 2024 11:46:04 +0200 Subject: [PATCH 02/15] test the seeder changes --- tests/dragonfly/replication_test.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 8d160f4e8715..af9aceb97f2e 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -18,6 +18,7 @@ from . import dfly_args from .proxy import Proxy from .seeder import StaticSeeder +from .seeder import SeederBase ADMIN_PORT = 1211 @@ -61,6 +62,34 @@ async def wait_for_replicas_state(*clients, state="online", node_role="slave", t pytest.param( 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS ), + # Quick general test that replication is working + ( + 1, + 3 * [1], + dict( + key_target=1_000, + huge_value_percentage=10, + huge_value_size=4096 * 2, + collection_size=10, + types=SeederBase.BIG_VALUE_TYPES, + ), + 500, + True, + ), + # Big value + ( + 4, + [4, 4], + dict( + key_target=10_000, + huge_value_percentage=5, + huge_value_size=4096 * 4, + collection_size=50, + types=SeederBase.BIG_VALUE_TYPES, + ), + 500, + True, + ), ], ) @pytest.mark.parametrize("mode", [({}), ({"cache_mode": "true"})]) @@ -132,6 +161,12 @@ async def check(): # Check data after stable state stream await check() + if big_value: + info = await c_master.info() + preemptions = info["big_value_preemptions"] + logging.info(f"Preemptions {preemptions}") + assert preemptions > 0 + async def check_replica_finished_exec(c_replica: aioredis.Redis, m_offset): role = await c_replica.role() From 7d587396e24a1f392373aa2e3b45ae9a36c1f3d4 Mon Sep 17 00:00:00 2001 From: kostas Date: Mon, 2 Dec 2024 11:33:41 +0200 Subject: [PATCH 03/15] add defaults --- tests/dragonfly/replication_test.py | 28 ---------------------------- tests/dragonfly/seeder/__init__.py | 4 ++-- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index af9aceb97f2e..220eb6cd7587 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -62,34 +62,6 @@ async def wait_for_replicas_state(*clients, state="online", node_role="slave", t pytest.param( 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS ), - # Quick general test that replication is working - ( - 1, - 3 * [1], - dict( - key_target=1_000, - huge_value_percentage=10, - huge_value_size=4096 * 2, - collection_size=10, - types=SeederBase.BIG_VALUE_TYPES, - ), - 500, - True, - ), - # Big value - ( - 4, - [4, 4], - dict( - key_target=10_000, - huge_value_percentage=5, - huge_value_size=4096 * 4, - collection_size=50, - types=SeederBase.BIG_VALUE_TYPES, - ), - 500, - True, - ), ], ) @pytest.mark.parametrize("mode", [({}), ({"cache_mode": "true"})]) diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index 29e12cb1a0db..373906ce6314 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -138,8 +138,8 @@ def __init__( data_size=100, collection_size=None, types: typing.Optional[typing.List[str]] = None, - huge_value_percentage=0, - huge_value_size=0, + huge_value_percentage=5, + huge_value_size=16384, ): SeederBase.__init__(self, types) self.key_target = key_target From 0eeda722119c7a53dd07c563a57b983edab524b2 Mon Sep 17 00:00:00 2001 From: kostas Date: Mon, 2 Dec 2024 15:47:33 +0200 Subject: [PATCH 04/15] comments + metrics Signed-off-by: kostas --- tests/dragonfly/replication_test.py | 7 -- tests/dragonfly/seeder/__init__.py | 7 +- tests/dragonfly/seeder/script-generate.lua | 7 +- tests/dragonfly/seeder/script-genlib.lua | 105 ++++++++------------- 4 files changed, 49 insertions(+), 77 deletions(-) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 220eb6cd7587..8d160f4e8715 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -18,7 +18,6 @@ from . import dfly_args from .proxy import Proxy from .seeder import StaticSeeder -from .seeder import SeederBase ADMIN_PORT = 1211 @@ -133,12 +132,6 @@ async def check(): # Check data after stable state stream await check() - if big_value: - info = await c_master.info() - preemptions = info["big_value_preemptions"] - logging.info(f"Preemptions {preemptions}") - assert preemptions > 0 - async def check_replica_finished_exec(c_replica: aioredis.Redis, m_offset): role = await c_replica.role() diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index 373906ce6314..e5314c29edd3 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -204,8 +204,11 @@ async def _run_unit(client: aioredis.Redis, sha: str, unit: Unit, using_stopkey, unit.stop_key if using_stopkey else "", ] + args - unit.counter = await client.evalsha(sha, 0, *args) + result = await client.evalsha(sha, 0, *args) + result = result.split() + unit.counter = int(result[0]) + huge_entries = int(result[1]) logging.debug( - f"running unit {unit.prefix}/{unit.type} took {time.time() - s}, target {args[4+0]}" + f"running unit {unit.prefix}/{unit.type} took {time.time() - s}, target {args[4+0]}, huge entries {huge_entries}" ) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index 9cb7489a6073..5ab6e3c22cf7 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -29,6 +29,7 @@ local keys = LU_collect_keys(prefix, type) LG_funcs.init(data_size, collection_size, huge_value_percentage, huge_value_size) local addfunc = LG_funcs['add_' .. string.lower(type)] local modfunc = LG_funcs['mod_' .. string.lower(type)] +local huge_entries = LG_funcs["get_huge_entries"] local function action_add() local key = prefix .. tostring(key_counter) @@ -87,7 +88,7 @@ while true do -- update probability only every 10 iterations if counter % 10 == 0 then -- calculate intensity (not normalized probabilities) - -- please see attached plots in PR to undertand convergence + -- please see attached plots in PR to understand convergence -- https://github.com/dragonflydb/dragonfly/pull/2556 -- the add intensity is monotonically decreasing with keycount growing, @@ -125,4 +126,6 @@ if stop_key ~= '' then redis.call('DEL', stop_key) end -return key_counter +return tostring(key_counter) .. " " .. tostring(huge_entries()) +---return tostring(key_counter) +---return key_counter diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index 06c4df532e4d..22395ad9e216 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -8,14 +8,38 @@ function LG_funcs.init(dsize, csize, large_val_perc, large_val_sz) LG_funcs.huge_value_size = large_val_sz end +local huge_entries = 0 + local function huge_entry() - local perc = LG_funcs.huge_value_percentage / 100 + local ratio = LG_funcs.huge_value_percentage / 100 -- [0, 1] local rand = math.random() - local huge_entry = (perc > rand) + local huge_entry = (ratio > rand) return huge_entry end +local function randstr() + local str + if huge_entry() then + str = dragonfly.randstr(LG_funcs.huge_value_size) + huge_entries = huge_entries + 1 + else + str = dragonfly.randstr(LG_funcs.esize) + end + return str +end + +local function randstr_sequence() + local strs + if huge_entry() then + strs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) + huge_entries = huge_entries + 1 + else + strs = dragonfly.randstr(LG_funcs.csize, LG_funcs.csize) + end + return strs +end + -- strings -- store blobs of random chars @@ -38,14 +62,7 @@ end -- store list of random blobs of default container/element sizes function LG_funcs.add_list(key, huge_value) - local elements - if huge_entry() then - elements = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) - else - elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) - end - - redis.apcall('LPUSH', key, unpack(elements)) + redis.apcall('LPUSH', key, unpack(randstr_sequence())) end function LG_funcs.mod_list(key, huge_value) @@ -57,23 +74,9 @@ function LG_funcs.mod_list(key, huge_value) elseif action == 2 then redis.apcall('LPOP', key) elseif action == 3 then - local str - if huge_entry() then - str = dragonfly.randstr(LG_funcs.huge_value_size) - else - str = dragonfly.randstr(LG_funcs.esize) - end - - redis.apcall('LPUSH', key, str) + redis.apcall('LPUSH', key, randstr()) else - local str - if huge_entry() then - str = dragonfly.randstr(LG_funcs.huge_value_size) - else - str = dragonfly.randstr(LG_funcs.esize) - end - - redis.apcall('RPUSH', key, str) + redis.apcall('RPUSH', key, randstr()) end end @@ -92,16 +95,7 @@ function LG_funcs.add_set(key, keys) end redis.apcall('SDIFFSTORE', key, keys[i1], keys[i2]) else - local elements - if huge_entry() then - -- Hard coded 10 here, meaning up to 10 huge entries per set - -- TODO make this configurable - elements = dragonfly.randstr(LG_funcs.large_val_sz, 10) - else - elements = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) - end - - redis.apcall('SADD', key, unpack(elements)) + redis.apcall('SADD', key, unpack(randstr_sequence())) end end @@ -110,14 +104,7 @@ function LG_funcs.mod_set(key) if math.random() < 0.5 then redis.apcall('SPOP', key) else - local rand_str - if huge_entry() then - rand_str = dragonfly.randstr(LG_funcs.huge_value_size) - else - rand_str = dragonfly.randstr(LG_funcs.esize) - end - - redis.apcall('SADD', key, rand_str) + redis.apcall('SADD', key, randstr()) end end @@ -130,6 +117,7 @@ function LG_funcs.add_hash(key) local blobs if huge_entry() then blobs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize / 2) + huge_entries = huge_entries + 1 else blobs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize / 2) end @@ -151,14 +139,7 @@ function LG_funcs.mod_hash(key) if idx % 2 == 1 then redis.apcall('HINCRBY', key, tostring(idx), 1) else - local str - if huge_entry() then - str = dragonfly.randstr(LG_funcs.large_val_sz) - else - str = dragonfly.randstr(LG_funcs.esize) - end - - redis.apcall('HSET', key, tostring(idx), str) + redis.apcall('HSET', key, tostring(idx), randstr()) end end @@ -166,12 +147,7 @@ end function LG_funcs.add_zset(key, keys) -- TODO: We don't support ZDIFFSTORE - local blobs - if huge_entry() then - blobs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) - else - blobs = dragonfly.randstr(LG_funcs.csize, LG_funcs.csize) - end + local blobs = randstr_sequence() local ztable = {} for i = 1, LG_funcs.csize do @@ -184,14 +160,7 @@ end function LG_funcs.mod_zset(key, dbsize) local action = math.random(1, 4) if action <= 2 then - local str - if huge_entry() then - str = dragonfly.randstr(LG_funcs.large_val_sz) - else - str = dragonfly.randstr(LG_funcs.esize) - end - - redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), str) + redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), randstr()) elseif action == 3 then redis.apcall('ZPOPMAX', key) else @@ -224,3 +193,7 @@ function LG_funcs.mod_json(key, dbsize) redis.apcall('JSON.NUMINCRBY', key, '$.counters[' .. math.random(LG_funcs.csize ) .. ']', 1) end end + +function LG_funcs.get_huge_entries() + return huge_entries +end From a55c79ae766c534596673aaa27d774c99d29d116 Mon Sep 17 00:00:00 2001 From: kostas Date: Mon, 2 Dec 2024 15:49:50 +0200 Subject: [PATCH 05/15] remove unused --- tests/dragonfly/seeder/script-generate.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index 5ab6e3c22cf7..207a8ad0c3a3 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -127,5 +127,3 @@ if stop_key ~= '' then end return tostring(key_counter) .. " " .. tostring(huge_entries()) ----return tostring(key_counter) ----return key_counter From 089c8ade2d3141f1eaad50977414e0ad1dbbd357 Mon Sep 17 00:00:00 2001 From: kostas Date: Mon, 2 Dec 2024 17:34:54 +0200 Subject: [PATCH 06/15] fixes --- tests/dragonfly/seeder/script-genlib.lua | 2 +- tests/dragonfly/seeder_test.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index 22395ad9e216..0f723243be55 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -35,7 +35,7 @@ local function randstr_sequence() strs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) huge_entries = huge_entries + 1 else - strs = dragonfly.randstr(LG_funcs.csize, LG_funcs.csize) + strs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) end return strs end diff --git a/tests/dragonfly/seeder_test.py b/tests/dragonfly/seeder_test.py index 5086cc399495..3c1913f88340 100644 --- a/tests/dragonfly/seeder_test.py +++ b/tests/dragonfly/seeder_test.py @@ -29,7 +29,15 @@ async def check_list(): await async_client.flushall() - s = Seeder(units=1, key_target=10, data_size=10_000, collection_size=1, types=["LIST"]) + s = Seeder( + units=1, + key_target=10, + data_size=10_000, + collection_size=1, + types=["LIST"], + huge_value_percentage=0, + huge_value_size=0, + ) await s.run(async_client) await check_list() From a2020c4714cad7440ee7a492971a55d6217670af Mon Sep 17 00:00:00 2001 From: kostas Date: Tue, 3 Dec 2024 12:21:02 +0200 Subject: [PATCH 07/15] fixes --- tests/dragonfly/seeder/__init__.py | 9 ++-- tests/dragonfly/seeder/script-generate.lua | 24 +++++++++- tests/dragonfly/seeder/script-genlib.lua | 54 +++++++++++----------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index e5314c29edd3..2969997b0042 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -138,8 +138,8 @@ def __init__( data_size=100, collection_size=None, types: typing.Optional[typing.List[str]] = None, - huge_value_percentage=5, - huge_value_size=16384, + huge_value_percentage=3, + huge_value_size=8192, ): SeederBase.__init__(self, types) self.key_target = key_target @@ -207,8 +207,9 @@ async def _run_unit(client: aioredis.Redis, sha: str, unit: Unit, using_stopkey, result = await client.evalsha(sha, 0, *args) result = result.split() unit.counter = int(result[0]) - huge_entries = int(result[1]) + huge_keys = int(result[1]) + huge_entries = int(result[2]) logging.debug( - f"running unit {unit.prefix}/{unit.type} took {time.time() - s}, target {args[4+0]}, huge entries {huge_entries}" + f"running unit {unit.prefix}/{unit.type} took {time.time() - s}, target {args[4+0]}, huge keys {huge_keys} with total huge entries {huge_entries}" ) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index 207a8ad0c3a3..74a5f722833e 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -31,12 +31,32 @@ local addfunc = LG_funcs['add_' .. string.lower(type)] local modfunc = LG_funcs['mod_' .. string.lower(type)] local huge_entries = LG_funcs["get_huge_entries"] +local huge_keys = 0 + +local function huge_entry() + local ratio = LG_funcs.huge_value_percentage / 100 + -- [0, 1] + local rand = math.random() + local huge_entry = (ratio > rand) + return huge_entry +end + local function action_add() local key = prefix .. tostring(key_counter) + local op_type = string.lower(type) + local is_huge = false + if op_type ~= "string" and op_type ~= "json" then + is_huge = huge_entry() + end + key_counter = key_counter + 1 + if is_huge then + huge_keys = huge_keys + 1 + end - addfunc(key, keys) table.insert(keys, key) + keys[key] = is_huge + addfunc(key, keys) end local function action_mod() @@ -126,4 +146,4 @@ if stop_key ~= '' then redis.call('DEL', stop_key) end -return tostring(key_counter) .. " " .. tostring(huge_entries()) +return tostring(key_counter) .. " " .. tostring(huge_keys) .. " " .. tostring(huge_entries()) diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index 0f723243be55..aa53e18dec65 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -10,17 +10,9 @@ end local huge_entries = 0 -local function huge_entry() - local ratio = LG_funcs.huge_value_percentage / 100 - -- [0, 1] - local rand = math.random() - local huge_entry = (ratio > rand) - return huge_entry -end - -local function randstr() +local function randstr(huge_entry) local str - if huge_entry() then + if huge_entry then str = dragonfly.randstr(LG_funcs.huge_value_size) huge_entries = huge_entries + 1 else @@ -29,9 +21,9 @@ local function randstr() return str end -local function randstr_sequence() +local function randstr_sequence(huge_entry) local strs - if huge_entry() then + if huge_entry then strs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) huge_entries = huge_entries + 1 else @@ -61,22 +53,24 @@ end -- lists -- store list of random blobs of default container/element sizes -function LG_funcs.add_list(key, huge_value) - redis.apcall('LPUSH', key, unpack(randstr_sequence())) +function LG_funcs.add_list(key, keys) + local is_huge = keys[key] + redis.apcall('LPUSH', key, unpack(randstr_sequence(is_huge))) end -function LG_funcs.mod_list(key, huge_value) +function LG_funcs.mod_list(key, keys) -- equally likely pops and pushes, we rely on the list size being large enough -- to "highly likely" not get emptied out by consequitve pops + local is_huge = keys[key] local action = math.random(1, 4) if action == 1 then redis.apcall('RPOP', key) elseif action == 2 then redis.apcall('LPOP', key) elseif action == 3 then - redis.apcall('LPUSH', key, randstr()) + redis.apcall('LPUSH', key, randstr(is_huge)) else - redis.apcall('RPUSH', key, randstr()) + redis.apcall('RPUSH', key, randstr(is_huge)) end end @@ -95,16 +89,18 @@ function LG_funcs.add_set(key, keys) end redis.apcall('SDIFFSTORE', key, keys[i1], keys[i2]) else - redis.apcall('SADD', key, unpack(randstr_sequence())) + local is_huge = keys[key] + redis.apcall('SADD', key, unpack(randstr_sequence(is_huge))) end end -function LG_funcs.mod_set(key) +function LG_funcs.mod_set(key, keys) -- equally likely pops and additions if math.random() < 0.5 then redis.apcall('SPOP', key) else - redis.apcall('SADD', key, randstr()) + local is_huge = keys[key] + redis.apcall('SADD', key, randstr(is_huge)) end end @@ -113,9 +109,10 @@ end -- store {to_string(i): value for i in [1, csize]}, -- where `value` is a random string for even indices and a number for odd indices -function LG_funcs.add_hash(key) +function LG_funcs.add_hash(key, keys) local blobs - if huge_entry() then + local is_huge = keys[key] + if is_huge then blobs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize / 2) huge_entries = huge_entries + 1 else @@ -134,12 +131,13 @@ function LG_funcs.add_hash(key) redis.apcall('HSET', key, unpack(htable)) end -function LG_funcs.mod_hash(key) +function LG_funcs.mod_hash(key, keys) local idx = math.random(LG_funcs.csize) if idx % 2 == 1 then redis.apcall('HINCRBY', key, tostring(idx), 1) else - redis.apcall('HSET', key, tostring(idx), randstr()) + local is_huge = keys[key] + redis.apcall('HSET', key, tostring(idx), randstr(is_huge)) end end @@ -147,7 +145,8 @@ end function LG_funcs.add_zset(key, keys) -- TODO: We don't support ZDIFFSTORE - local blobs = randstr_sequence() + local is_huge = keys[key] + local blobs = randstr_sequence(is_huge) local ztable = {} for i = 1, LG_funcs.csize do @@ -157,10 +156,11 @@ function LG_funcs.add_zset(key, keys) redis.apcall('ZADD', key, unpack(ztable)) end -function LG_funcs.mod_zset(key, dbsize) +function LG_funcs.mod_zset(key, keys) local action = math.random(1, 4) if action <= 2 then - redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), randstr()) + local is_huge = keys[key] + redis.apcall('ZADD', key, math.random(0, LG_funcs.csize * 2), randstr(is_huge)) elseif action == 3 then redis.apcall('ZPOPMAX', key) else From 473ee5c0b5b5e7935075d72992570469087f9cb8 Mon Sep 17 00:00:00 2001 From: kostas Date: Tue, 3 Dec 2024 14:01:09 +0200 Subject: [PATCH 08/15] small text --- tests/dragonfly/replication_test.py | 26 +++++++++++++------------- tests/dragonfly/seeder/__init__.py | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 8d160f4e8715..4425a2e0f13a 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -48,19 +48,19 @@ async def wait_for_replicas_state(*clients, state="online", node_role="slave", t [ # Quick general test that replication is working (1, 3 * [1], dict(key_target=1_000), 500, False), - (4, [4, 4], dict(key_target=10_000), 1_000, False), - pytest.param(6, [6, 6, 6], dict(key_target=100_000), 20_000, False, marks=M_OPT), - # Skewed tests with different thread ratio - pytest.param(8, 6 * [1], dict(key_target=5_000), 2_000, False, marks=M_SLOW), - pytest.param(2, [8, 8], dict(key_target=10_000), 2_000, False, marks=M_SLOW), - # Test with big value size - pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, False, marks=M_SLOW), - # Test with big value and big value serialization - pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, True, marks=M_SLOW), - # Stress test - pytest.param( - 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS - ), + # (4, [4, 4], dict(key_target=10_000), 1_000, False), + # pytest.param(6, [6, 6, 6], dict(key_target=100_000), 20_000, False, marks=M_OPT), + # # Skewed tests with different thread ratio + # pytest.param(8, 6 * [1], dict(key_target=5_000), 2_000, False, marks=M_SLOW), + # pytest.param(2, [8, 8], dict(key_target=10_000), 2_000, False, marks=M_SLOW), + # # Test with big value size + # pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, False, marks=M_SLOW), + # # Test with big value and big value serialization + # pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, True, marks=M_SLOW), + # # Stress test + # pytest.param( + # 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS + # ), ], ) @pytest.mark.parametrize("mode", [({}), ({"cache_mode": "true"})]) diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index 2969997b0042..66e0d775a1bf 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -138,8 +138,8 @@ def __init__( data_size=100, collection_size=None, types: typing.Optional[typing.List[str]] = None, - huge_value_percentage=3, - huge_value_size=8192, + huge_value_percentage=1, + huge_value_size=4096, ): SeederBase.__init__(self, types) self.key_target = key_target From edb2ed6a27b6571eb0238beb4e8ab1b37bcaa332 Mon Sep 17 00:00:00 2001 From: kostas Date: Wed, 4 Dec 2024 09:42:52 +0200 Subject: [PATCH 09/15] fixes Signed-off-by: kostas --- tests/dragonfly/replication_test.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 4425a2e0f13a..8d160f4e8715 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -48,19 +48,19 @@ async def wait_for_replicas_state(*clients, state="online", node_role="slave", t [ # Quick general test that replication is working (1, 3 * [1], dict(key_target=1_000), 500, False), - # (4, [4, 4], dict(key_target=10_000), 1_000, False), - # pytest.param(6, [6, 6, 6], dict(key_target=100_000), 20_000, False, marks=M_OPT), - # # Skewed tests with different thread ratio - # pytest.param(8, 6 * [1], dict(key_target=5_000), 2_000, False, marks=M_SLOW), - # pytest.param(2, [8, 8], dict(key_target=10_000), 2_000, False, marks=M_SLOW), - # # Test with big value size - # pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, False, marks=M_SLOW), - # # Test with big value and big value serialization - # pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, True, marks=M_SLOW), - # # Stress test - # pytest.param( - # 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS - # ), + (4, [4, 4], dict(key_target=10_000), 1_000, False), + pytest.param(6, [6, 6, 6], dict(key_target=100_000), 20_000, False, marks=M_OPT), + # Skewed tests with different thread ratio + pytest.param(8, 6 * [1], dict(key_target=5_000), 2_000, False, marks=M_SLOW), + pytest.param(2, [8, 8], dict(key_target=10_000), 2_000, False, marks=M_SLOW), + # Test with big value size + pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, False, marks=M_SLOW), + # Test with big value and big value serialization + pytest.param(2, [2], dict(key_target=1_000, data_size=10_000), 100, True, marks=M_SLOW), + # Stress test + pytest.param( + 8, [8, 8], dict(key_target=1_000_000, units=16), 50_000, False, marks=M_STRESS + ), ], ) @pytest.mark.parametrize("mode", [({}), ({"cache_mode": "true"})]) From 2eae4e2e40a4f21fa85387f635075f416723dcd6 Mon Sep 17 00:00:00 2001 From: kostas Date: Wed, 4 Dec 2024 13:23:46 +0200 Subject: [PATCH 10/15] turn on allocation tracker --- tests/dragonfly/replication_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 8d160f4e8715..14cd6b52b867 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -81,6 +81,8 @@ async def test_replication_all( if big_value: args["serialization_max_chunk_size"] = 4096 + args["allocation_tracker"] = "1000000000:40000000000" + master = df_factory.create(admin_port=ADMIN_PORT, proactor_threads=t_master, **args) replicas = [ df_factory.create(admin_port=ADMIN_PORT + i + 1, proactor_threads=t) From b0e85fb291548109b267141e6f99a5eeec068d44 Mon Sep 17 00:00:00 2001 From: kostas Date: Wed, 4 Dec 2024 15:44:42 +0200 Subject: [PATCH 11/15] fix uncontrolled huge entries --- tests/dragonfly/replication_test.py | 2 -- tests/dragonfly/seeder/__init__.py | 6 +++++- tests/dragonfly/seeder/script-generate.lua | 3 ++- tests/dragonfly/seeder/script-genlib.lua | 5 +++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/dragonfly/replication_test.py b/tests/dragonfly/replication_test.py index 14cd6b52b867..8d160f4e8715 100644 --- a/tests/dragonfly/replication_test.py +++ b/tests/dragonfly/replication_test.py @@ -81,8 +81,6 @@ async def test_replication_all( if big_value: args["serialization_max_chunk_size"] = 4096 - args["allocation_tracker"] = "1000000000:40000000000" - master = df_factory.create(admin_port=ADMIN_PORT, proactor_threads=t_master, **args) replicas = [ df_factory.create(admin_port=ADMIN_PORT + i + 1, proactor_threads=t) diff --git a/tests/dragonfly/seeder/__init__.py b/tests/dragonfly/seeder/__init__.py index 66e0d775a1bf..069351d42437 100644 --- a/tests/dragonfly/seeder/__init__.py +++ b/tests/dragonfly/seeder/__init__.py @@ -138,8 +138,10 @@ def __init__( data_size=100, collection_size=None, types: typing.Optional[typing.List[str]] = None, - huge_value_percentage=1, + huge_value_percentage=2, huge_value_size=4096, + # 2 huge entries per container/key as default + huge_value_csize=2, ): SeederBase.__init__(self, types) self.key_target = key_target @@ -151,6 +153,7 @@ def __init__( self.huge_value_percentage = huge_value_percentage self.huge_value_size = huge_value_size + self.huge_value_csize = huge_value_csize self.units = [ Seeder.Unit( @@ -174,6 +177,7 @@ async def run(self, client: aioredis.Redis, target_ops=None, target_deviation=No self.collection_size, self.huge_value_percentage, self.huge_value_size, + self.huge_value_csize, ] sha = await client.script_load(Seeder._load_script("generate")) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index 74a5f722833e..0731a05310f5 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -21,12 +21,13 @@ local collection_size = tonumber(ARGV[9]) -- Probability of each key in key_target to be a big value local huge_value_percentage = tonumber(ARGV[10]) local huge_value_size = tonumber(ARGV[11]) +local huge_value_csize = tonumber(ARGV[12]) -- collect all keys belonging to this script -- assumes exclusive ownership local keys = LU_collect_keys(prefix, type) -LG_funcs.init(data_size, collection_size, huge_value_percentage, huge_value_size) +LG_funcs.init(data_size, collection_size, huge_value_percentage, huge_value_size, huge_value_csize) local addfunc = LG_funcs['add_' .. string.lower(type)] local modfunc = LG_funcs['mod_' .. string.lower(type)] local huge_entries = LG_funcs["get_huge_entries"] diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index aa53e18dec65..cc6b7b1cfbd2 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -1,11 +1,12 @@ local LG_funcs = {} -function LG_funcs.init(dsize, csize, large_val_perc, large_val_sz) +function LG_funcs.init(dsize, csize, large_val_perc, large_val_sz, huge_value_csize) LG_funcs.dsize = dsize LG_funcs.csize = csize LG_funcs.esize = math.ceil(dsize / csize) LG_funcs.huge_value_percentage = large_val_perc LG_funcs.huge_value_size = large_val_sz + LG_funcs.huge_value_csize = huge_value_csize end local huge_entries = 0 @@ -24,7 +25,7 @@ end local function randstr_sequence(huge_entry) local strs if huge_entry then - strs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.csize) + strs = dragonfly.randstr(LG_funcs.huge_value_size, LG_funcs.huge_value_csize) huge_entries = huge_entries + 1 else strs = dragonfly.randstr(LG_funcs.esize, LG_funcs.csize) From 215a74a1f39d59574532511def0f3a95640188e0 Mon Sep 17 00:00:00 2001 From: kostas Date: Wed, 4 Dec 2024 16:34:37 +0200 Subject: [PATCH 12/15] tune --- .github/actions/regression-tests/action.yml | 2 +- .github/workflows/ci.yml | 80 ++++++++++----------- tests/dragonfly/seeder/__init__.py | 6 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/actions/regression-tests/action.yml b/.github/actions/regression-tests/action.yml index 01fddc2ba602..1af770e46d2c 100644 --- a/.github/actions/regression-tests/action.yml +++ b/.github/actions/regression-tests/action.yml @@ -46,7 +46,7 @@ runs: export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors - timeout 50m pytest -m "${{inputs.filter}}" --durations=10 --color=yes --json-report --json-report-file=report.json dragonfly --log-cli-level=INFO || code=$? + timeout 50m pytest -m "${{inputs.filter}}" --durations=10 --color=yes --json-report --json-report-file=report.json dragonfly/replication_test.py -k "test_replication_all" --log-cli-level=INFO || code=$? # timeout returns 124 if we exceeded the timeout duration if [[ $code -eq 124 ]]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d721f3576a7b..718cad4c2924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,46 +129,46 @@ jobs: echo "disk space is:" df -h - - name: C++ Unit Tests - run: | - cd ${GITHUB_WORKSPACE}/build - echo Run ctest -V -L DFLY - GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 \ - FLAGS_list_experimental_v2=true timeout 20m ctest -V -L DFLY - - echo "Running tests with --force_epoll" - - # Create a rule that automatically prints stacktrace upon segfault - cat > ./init.gdb < ./init.gdb < Date: Wed, 4 Dec 2024 17:02:17 +0200 Subject: [PATCH 13/15] polish and remove ci changes --- .github/actions/regression-tests/action.yml | 2 +- .github/workflows/ci.yml | 80 ++++++++++----------- tests/dragonfly/seeder/__init__.py | 10 +-- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/.github/actions/regression-tests/action.yml b/.github/actions/regression-tests/action.yml index 1af770e46d2c..01fddc2ba602 100644 --- a/.github/actions/regression-tests/action.yml +++ b/.github/actions/regression-tests/action.yml @@ -46,7 +46,7 @@ runs: export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors - timeout 50m pytest -m "${{inputs.filter}}" --durations=10 --color=yes --json-report --json-report-file=report.json dragonfly/replication_test.py -k "test_replication_all" --log-cli-level=INFO || code=$? + timeout 50m pytest -m "${{inputs.filter}}" --durations=10 --color=yes --json-report --json-report-file=report.json dragonfly --log-cli-level=INFO || code=$? # timeout returns 124 if we exceeded the timeout duration if [[ $code -eq 124 ]]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 718cad4c2924..d721f3576a7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,46 +129,46 @@ jobs: echo "disk space is:" df -h - # - name: C++ Unit Tests - # run: | - # cd ${GITHUB_WORKSPACE}/build - # echo Run ctest -V -L DFLY - # GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 \ - # FLAGS_list_experimental_v2=true timeout 20m ctest -V -L DFLY - # - # echo "Running tests with --force_epoll" - # - # # Create a rule that automatically prints stacktrace upon segfault - # cat > ./init.gdb < ./init.gdb < 0: + msg = f"{msg}. Total huge keys added {huge_keys} with {args[11]} elements each. Total extra modified huge entries {huge_entries}." + + logging.debug(msg) From 0408086ce6296e1e307d88a80a34215c92bd1a23 Mon Sep 17 00:00:00 2001 From: kostas Date: Wed, 4 Dec 2024 18:04:18 +0200 Subject: [PATCH 14/15] fix zset --- tests/dragonfly/seeder/script-genlib.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/dragonfly/seeder/script-genlib.lua b/tests/dragonfly/seeder/script-genlib.lua index cc6b7b1cfbd2..fb497c17fc98 100644 --- a/tests/dragonfly/seeder/script-genlib.lua +++ b/tests/dragonfly/seeder/script-genlib.lua @@ -150,7 +150,13 @@ function LG_funcs.add_zset(key, keys) local blobs = randstr_sequence(is_huge) local ztable = {} - for i = 1, LG_funcs.csize do + + local limit = LG_funcs.csize + if is_huge then + limit = LG_funcs.huge_value_csize + end + + for i = 1, limit do ztable[i * 2 - 1] = tostring(i) ztable[i * 2] = blobs[i] end From 141dc711bbfdd38509fe276d429297a57d9d82a2 Mon Sep 17 00:00:00 2001 From: kostas Date: Thu, 5 Dec 2024 10:08:50 +0200 Subject: [PATCH 15/15] add comments --- tests/dragonfly/seeder/script-generate.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/dragonfly/seeder/script-generate.lua b/tests/dragonfly/seeder/script-generate.lua index 0731a05310f5..32b2e1d4ccfa 100644 --- a/tests/dragonfly/seeder/script-generate.lua +++ b/tests/dragonfly/seeder/script-generate.lua @@ -46,6 +46,8 @@ local function action_add() local key = prefix .. tostring(key_counter) local op_type = string.lower(type) local is_huge = false + -- `string` and `json` huge entries are not supported so + -- we don't roll a dice to decide if they are huge or not if op_type ~= "string" and op_type ~= "json" then is_huge = huge_entry() end