Skip to content

Commit ceeb7cd

Browse files
romangeromange
committed
chore: Add initial bindings for QList in list_family
The feature is guarded by list_experimental_v2 flag, which is disabled. Signed-off-by: Roman Gershman <[email protected]>
1 parent 1819e51 commit ceeb7cd

File tree

5 files changed

+99
-48
lines changed

5 files changed

+99
-48
lines changed

‎src/core/compact_object.cc

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@ extern "C" {
2727
#include "base/pod_array.h"
2828
#include "core/bloom.h"
2929
#include "core/detail/bitpacking.h"
30+
#include "core/qlist.h"
3031
#include "core/sorted_map.h"
3132
#include "core/string_map.h"
3233
#include "core/string_set.h"
3334

34-
ABSL_RETIRED_FLAG(bool, use_set2, true, "If true use DenseSet for an optimized set data structure");
35-
3635
ABSL_FLAG(bool, experimental_flat_json, false, "If true uses flat json implementation.");
3736

3837
namespace dfly {
@@ -67,6 +66,19 @@ inline void FreeObjSet(unsigned encoding, void* ptr, MemoryResource* mr) {
6766
}
6867
}
6968

69+
void FreeList(unsigned encoding, void* ptr, MemoryResource* mr) {
70+
switch (encoding) {
71+
case OBJ_ENCODING_QUICKLIST:
72+
quicklistRelease((quicklist*)ptr);
73+
break;
74+
case kEncodingQL2:
75+
CompactObj::DeleteMR<QList>(ptr);
76+
break;
77+
default:
78+
LOG(FATAL) << "Unknown list encoding type";
79+
}
80+
}
81+
7082
size_t MallocUsedSet(unsigned encoding, void* ptr) {
7183
switch (encoding) {
7284
case kEncodingStrMap2: {
@@ -288,8 +300,9 @@ size_t RobjWrapper::MallocUsed() const {
288300
CHECK_EQ(OBJ_ENCODING_RAW, encoding_);
289301
return InnerObjMallocUsed();
290302
case OBJ_LIST:
291-
DCHECK_EQ(encoding_, OBJ_ENCODING_QUICKLIST);
292-
return QlMAllocSize((quicklist*)inner_obj_);
303+
if (encoding_ == OBJ_ENCODING_QUICKLIST)
304+
return QlMAllocSize((quicklist*)inner_obj_);
305+
return ((QList*)inner_obj_)->MallocUsed();
293306
case OBJ_SET:
294307
return MallocUsedSet(encoding_, inner_obj_);
295308
case OBJ_HASH:
@@ -312,7 +325,9 @@ size_t RobjWrapper::Size() const {
312325
DCHECK_EQ(OBJ_ENCODING_RAW, encoding_);
313326
return sz_;
314327
case OBJ_LIST:
315-
return quicklistCount((quicklist*)inner_obj_);
328+
if (encoding_ == OBJ_ENCODING_QUICKLIST)
329+
return quicklistCount((quicklist*)inner_obj_);
330+
return ((QList*)inner_obj_)->Size();
316331
case OBJ_ZSET: {
317332
switch (encoding_) {
318333
case OBJ_ENCODING_SKIPLIST: {
@@ -367,8 +382,7 @@ void RobjWrapper::Free(MemoryResource* mr) {
367382
mr->deallocate(inner_obj_, 0, 8); // we do not keep the allocated size.
368383
break;
369384
case OBJ_LIST:
370-
CHECK_EQ(encoding_, OBJ_ENCODING_QUICKLIST);
371-
quicklistRelease((quicklist*)inner_obj_);
385+
FreeList(encoding_, inner_obj_, mr);
372386
break;
373387
case OBJ_SET:
374388
FreeObjSet(encoding_, inner_obj_, mr);

‎src/core/compact_object.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
namespace dfly {
2020

2121
constexpr unsigned kEncodingIntSet = 0;
22-
constexpr unsigned kEncodingStrMap = 1; // for set/map encodings of strings
2322
constexpr unsigned kEncodingStrMap2 = 2; // for set/map encodings of strings using DenseSet
23+
constexpr unsigned kEncodingQL2 = 1;
2424
constexpr unsigned kEncodingListPack = 3;
2525
constexpr unsigned kEncodingJsonCons = 0;
2626
constexpr unsigned kEncodingJsonFlat = 1;

‎src/server/list_family.cc

Lines changed: 76 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ extern "C" {
1414

1515
#include "base/flags.h"
1616
#include "base/logging.h"
17+
#include "core/qlist.h"
1718
#include "server/blocking_controller.h"
1819
#include "server/command_registry.h"
1920
#include "server/conn_context.h"
@@ -59,6 +60,8 @@ ABSL_FLAG(int32_t, list_max_listpack_size, -2, "Maximum listpack size, default i
5960
*/
6061

6162
ABSL_FLAG(int32_t, list_compress_depth, 0, "Compress depth of the list. Default is no compression");
63+
ABSL_FLAG(bool, list_experimental_v2, false,
64+
"Compress depth of the list. Default is no compression");
6265

6366
namespace dfly {
6467

@@ -73,6 +76,10 @@ quicklist* GetQL(const PrimeValue& mv) {
7376
return (quicklist*)mv.RObjPtr();
7477
}
7578

79+
QList* GetQLV2(const PrimeValue& mv) {
80+
return (QList*)mv.RObjPtr();
81+
}
82+
7683
void* listPopSaver(unsigned char* data, size_t sz) {
7784
return new string((char*)data, sz);
7885
}
@@ -265,26 +272,46 @@ OpResult<uint32_t> OpPush(const OpArgs& op_args, std::string_view key, ListDir d
265272
res = std::move(*op_res);
266273
}
267274

268-
quicklist* ql = nullptr;
275+
size_t len = 0;
269276
DVLOG(1) << "OpPush " << key << " new_key " << res.is_new;
277+
quicklist* ql = nullptr;
278+
QList* ql_v2 = nullptr;
270279

271280
if (res.is_new) {
272-
ql = quicklistCreate();
273-
quicklistSetOptions(ql, GetFlag(FLAGS_list_max_listpack_size),
274-
GetFlag(FLAGS_list_compress_depth));
275-
res.it->second.InitRobj(OBJ_LIST, OBJ_ENCODING_QUICKLIST, ql);
281+
if (absl::GetFlag(FLAGS_list_experimental_v2)) {
282+
ql_v2 = CompactObj::AllocateMR<QList>(GetFlag(FLAGS_list_max_listpack_size),
283+
GetFlag(FLAGS_list_compress_depth));
284+
res.it->second.InitRobj(OBJ_LIST, kEncodingQL2, ql_v2);
285+
} else {
286+
ql = quicklistCreate();
287+
quicklistSetOptions(ql, GetFlag(FLAGS_list_max_listpack_size),
288+
GetFlag(FLAGS_list_compress_depth));
289+
res.it->second.InitRobj(OBJ_LIST, OBJ_ENCODING_QUICKLIST, ql);
290+
}
276291
} else {
277292
if (res.it->second.ObjType() != OBJ_LIST)
278293
return OpStatus::WRONG_TYPE;
279-
ql = GetQL(res.it->second);
294+
if (res.it->second.Encoding() == kEncodingQL2) {
295+
ql_v2 = GetQLV2(res.it->second);
296+
} else {
297+
ql = GetQL(res.it->second);
298+
}
280299
}
281300

282-
// Left push is LIST_HEAD.
283-
int pos = (dir == ListDir::LEFT) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
284-
285-
for (string_view v : vals) {
286-
auto vsds = WrapSds(v);
287-
quicklistPush(ql, vsds, sdslen(vsds), pos);
301+
if (ql) {
302+
// Left push is LIST_HEAD.
303+
int pos = (dir == ListDir::LEFT) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
304+
for (string_view v : vals) {
305+
auto vsds = WrapSds(v);
306+
quicklistPush(ql, vsds, sdslen(vsds), pos);
307+
}
308+
len = quicklistCount(ql);
309+
} else {
310+
QList::Where where = (dir == ListDir::LEFT) ? QList::HEAD : QList::TAIL;
311+
for (string_view v : vals) {
312+
ql_v2->Push(v, where);
313+
}
314+
len = ql_v2->Size();
288315
}
289316

290317
if (res.is_new) {
@@ -305,7 +332,7 @@ OpResult<uint32_t> OpPush(const OpArgs& op_args, std::string_view key, ListDir d
305332
RecordJournal(op_args, command, mapped, 2);
306333
}
307334

308-
return quicklistCount(ql);
335+
return len;
309336
}
310337

311338
OpResult<StringVec> OpPop(const OpArgs& op_args, string_view key, ListDir dir, uint32_t count,
@@ -320,26 +347,26 @@ OpResult<StringVec> OpPop(const OpArgs& op_args, string_view key, ListDir dir, u
320347

321348
auto it = it_res->it;
322349
quicklist* ql = GetQL(it->second);
323-
350+
auto prev_len = quicklistCount(ql);
324351
StringVec res;
325-
if (quicklistCount(ql) < count) {
326-
count = quicklistCount(ql);
352+
if (prev_len < count) {
353+
count = prev_len;
327354
}
328-
res.reserve(count);
329355

330356
if (return_results) {
331-
for (unsigned i = 0; i < count; ++i) {
332-
res.push_back(ListPop(dir, ql));
333-
}
334-
} else {
335-
for (unsigned i = 0; i < count; ++i) {
336-
ListPop(dir, ql);
357+
res.reserve(count);
358+
}
359+
360+
for (unsigned i = 0; i < count; ++i) {
361+
string val = ListPop(dir, ql);
362+
if (return_results) {
363+
res.push_back(std::move(val));
337364
}
338365
}
339366

340367
it_res->post_updater.Run();
341368

342-
if (quicklistCount(ql) == 0) {
369+
if (count == prev_len) {
343370
CHECK(db_slice.Del(op_args.db_cntx, it));
344371
}
345372

@@ -418,31 +445,43 @@ OpResult<uint32_t> OpLen(const OpArgs& op_args, std::string_view key) {
418445
if (!res)
419446
return res.status();
420447

421-
quicklist* ql = GetQL(res.value()->second);
448+
if (res.value()->second.Encoding() == kEncodingQL2) {
449+
QList* ql = GetQLV2(res.value()->second);
450+
return ql->Size();
451+
}
422452

453+
quicklist* ql = GetQL(res.value()->second);
423454
return quicklistCount(ql);
424455
}
425456

426457
OpResult<string> OpIndex(const OpArgs& op_args, std::string_view key, long index) {
427458
auto res = op_args.GetDbSlice().FindReadOnly(op_args.db_cntx, key, OBJ_LIST);
428459
if (!res)
429460
return res.status();
430-
quicklist* ql = GetQL(res.value()->second);
431-
quicklistEntry entry = container_utils::QLEntry();
432-
quicklistIter* iter = quicklistGetIteratorAtIdx(ql, AL_START_TAIL, index);
433-
if (!iter)
434-
return OpStatus::KEY_NOTFOUND;
435461

436-
quicklistNext(iter, &entry);
437462
string str;
438-
439-
if (entry.value) {
440-
str.assign(reinterpret_cast<char*>(entry.value), entry.sz);
463+
if (res.value()->second.Encoding() == kEncodingQL2) {
464+
QList* ql = GetQLV2(res.value()->second);
465+
auto it = ql->GetIterator(index);
466+
if (!it.Next())
467+
return OpStatus::KEY_NOTFOUND;
468+
str = it.Get().to_string();
441469
} else {
442-
str = absl::StrCat(entry.longval);
443-
}
444-
quicklistReleaseIterator(iter);
470+
quicklist* ql = GetQL(res.value()->second);
471+
quicklistEntry entry = container_utils::QLEntry();
472+
quicklistIter* iter = quicklistGetIteratorAtIdx(ql, AL_START_TAIL, index);
473+
if (!iter)
474+
return OpStatus::KEY_NOTFOUND;
475+
476+
quicklistNext(iter, &entry);
445477

478+
if (entry.value) {
479+
str.assign(reinterpret_cast<char*>(entry.value), entry.sz);
480+
} else {
481+
str = absl::StrCat(entry.longval);
482+
}
483+
quicklistReleaseIterator(iter);
484+
}
446485
return str;
447486
}
448487

‎src/server/rdb_save.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ uint8_t RdbObjectType(const PrimeValue& pv) {
178178
case OBJ_SET:
179179
if (compact_enc == kEncodingIntSet)
180180
return RDB_TYPE_SET_INTSET;
181-
else if (compact_enc == kEncodingStrMap || compact_enc == kEncodingStrMap2) {
181+
else if (compact_enc == kEncodingStrMap2) {
182182
if (((StringSet*)pv.RObjPtr())->ExpirationUsed())
183183
return RDB_TYPE_SET_WITH_EXPIRY;
184184
else

‎src/server/set_family.cc

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ extern "C" {
2626
#include "server/journal/journal.h"
2727
#include "server/transaction.h"
2828

29-
ABSL_DECLARE_FLAG(bool, use_set2);
30-
3129
namespace dfly {
3230

3331
using namespace facade;

0 commit comments

Comments
 (0)