chore(tiering): External alloc free page chaining

Signed-off-by: Vladislav Oleshko <[email protected]>

src/server/tiering/external_alloc.cc

@@ -143,10 +143,13 @@ struct Page {
   // can be computed via free_blocks.count().
   uint16_t available;  // in number of blocks.
 
+  bool free_chained;  // whether it's present in the free_pages_ list
+  Page* next_free;    // next page in the free_pages_ list
+
   // We can not use c'tor because we use the trick in segment where we allocate more pages
   // than SegmentDescr declares.
   void Reset(uint8_t new_id) {
-    static_assert(sizeof(Page) == 40);
+    static_assert(sizeof(Page) == 56);
 
     memset(&id, 0, sizeof(Page) - offsetof(Page, id));
     id = new_id;
@@ -174,6 +177,9 @@ void Page::Init(PageClass pc, BinIdx bin_id) {
   for (unsigned i = 0; i < available; ++i) {
     free_blocks.set(i, true);
   }
+
+  free_chained = false;
+  next_free = nullptr;
 }
 
 PageClass ClassFromSize(size_t size) {
@@ -343,8 +349,10 @@ int64_t ExternalAllocator::Malloc(size_t sz) {
     page = FindPage(pc);
     if (!page)
       return -int64_t(kSegmentSize);
+
     free_pages_[bin_idx] = page;
     page->Init(pc, bin_idx);
+    page->free_chained = true;
   }
 
   DCHECK(page->available);
@@ -355,6 +363,11 @@ int64_t ExternalAllocator::Malloc(size_t sz) {
   --page->available;
   allocated_bytes_ += ToBlockSize(page->block_size_bin);
 
+  if (page->available == 0 && page->next_free != nullptr) {
+    page->free_chained = false;
+    free_pages_[bin_idx] = page->next_free;
+  }
+
   SegmentDescr* seg = ToSegDescr(page);
   return seg->BlockOffset(page, pos);
 }
@@ -386,6 +399,11 @@ void ExternalAllocator::Free(size_t offset, size_t sz) {
   DCHECK_EQ(page->available, page->free_blocks.count());
   if (page->available == blocks_num) {
     FreePage(page, seg, block_size);
+  } else if (page->available == 1) {
+    uint8_t bin_idx = ToBinIdx(sz);
+    page->free_chained = true;
+    page->next_free = free_pages_[bin_idx];
+    free_pages_[bin_idx] = page;
   }
   allocated_bytes_ -= block_size;
 }
@@ -478,12 +496,23 @@ void ExternalAllocator::FreePage(Page* page, SegmentDescr* owner, size_t block_s
   BinIdx bidx = ToBinIdx(block_size);
 
   // Remove fast allocation reference.
-  if (free_pages_[bidx] == page) {
-    free_pages_[bidx] = &empty_page;
+  if (page->free_chained) {
+    if (free_pages_[bidx] == page) {
+      free_pages_[bidx] = &empty_page;
+    } else {
+      for (auto* cur = free_pages_[bidx]; cur != nullptr; cur = cur->next_free) {
+        if (cur->next_free == page) {
+          cur->next_free = page->next_free;
+          break;
+        }
+      }
+    }
   }
 
   page->segment_inuse = 0;
   page->available = 0;
+  page->free_chained = false;
+  page->next_free = nullptr;
 
   if (!owner->HasFreePages()) {
     // Segment was fully booked but now it has a free page.

src/server/tiering/external_alloc.h

@@ -94,8 +94,8 @@ class ExternalAllocator {
 
   static SegmentDescr* ToSegDescr(Page*);
 
-  SegmentDescr* sq_[2];  // map: PageClass -> free Segment.
-  Page* free_pages_[detail::kNumFreePages];
+  SegmentDescr* sq_[2];                      // map: PageClass -> free Segment.
+  Page* free_pages_[detail::kNumFreePages];  // intrusive linked lists of pages with free blocks
 
   // A segment for each 256MB range. To get a segment id from the offset, shift right by 28.
   std::vector<SegmentDescr*> segments_;

src/server/tiering/external_alloc_test.cc

@@ -51,16 +51,18 @@ TEST_F(ExternalAllocatorTest, Basic) {
 
   ext_alloc_.AddStorage(0, kSegSize);
   EXPECT_EQ(0, ext_alloc_.Malloc(kMinBlockSize - 96));         //  page0: 1
-  EXPECT_EQ(kMinBlockSize, ext_alloc_.Malloc(kMinBlockSize));  // page0: 2
+  EXPECT_EQ(kMinBlockSize, ext_alloc_.Malloc(kMinBlockSize));  //  page0: 2
+
   constexpr auto kAnotherLen = kMinBlockSize * 2 - 10;
   size_t offset2 = ext_alloc_.Malloc(kAnotherLen);  // page1: 1
   EXPECT_EQ(offset2, 1_MB);                         // another page.
 
   ext_alloc_.Free(offset2, kAnotherLen);         // should return the page to the segment.
   EXPECT_EQ(offset2, ext_alloc_.Malloc(16_KB));  // another page.  page1: 1
 
-  ext_alloc_.Free(0, kMinBlockSize - 96);              // page0: 1
-  ext_alloc_.Free(kMinBlockSize, kMinBlockSize);       // page0: 0
+  ext_alloc_.Free(0, kMinBlockSize - 96);         // page0: 1
+  ext_alloc_.Free(kMinBlockSize, kMinBlockSize);  // page0: 0
+
   EXPECT_EQ(0, ext_alloc_.Malloc(kMinBlockSize * 2));  // page0
 }
 
@@ -117,4 +119,31 @@ TEST_F(ExternalAllocatorTest, Classes) {
   EXPECT_EQ(1_MB + 4_KB, ExternalAllocator::GoodSize(1_MB + 1));
 }
 
+// Fill up the allocator until it has to grow, remove 90% and make sure it has free space even with
+// extreme fragmentation
+TEST_F(ExternalAllocatorTest, EmptyFull) {
+  const int kAllocSize = kMinBlockSize;
+  ext_alloc_.AddStorage(0, 2 * kSegSize);
+
+  // Fill up the allocator
+  vector<int64_t> offsets;
+  int64_t offset;
+  do {
+    offset = ext_alloc_.Malloc(kAllocSize);
+    if (offset >= 0)
+      offsets.push_back(offset);
+  } while (offset >= 0);
+
+  // Keep only 10%, free 90%
+  for (size_t i = 0; i < offsets.size(); i++) {
+    if (i % 10 == 0)
+      continue;
+    ext_alloc_.Free(offsets[i], kAllocSize);
+  }
+
+  // Expect to succeed adding 10% without growing
+  for (size_t i = 0; i < offsets.size() / 10; i++)
+    EXPECT_GT(ext_alloc_.Malloc(kAllocSize), 0u);
+}
+
 }  // namespace dfly::tiering