Skip to content

Commit

Permalink
force GC, no cache during migration, auto heap profile (#19)
Browse files Browse the repository at this point in the history
* force GC, no cache during migration, auto heap profile

* resolve a potential deadlock from racing between reset and stop

* fix small lint issue

* remove logs and pprof logic

* remove unused libraries

* add comment explaining the reason for RAM optimizations
  • Loading branch information
p0mvn committed Feb 28, 2022
1 parent 543206b commit e44a79a
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 6 deletions.
43 changes: 41 additions & 2 deletions mutable_tree.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ import (
"bytes"
"crypto/sha256"
"fmt"
"runtime"
"sort"
"sync"
"time"

"github.com/pkg/errors"

Expand Down Expand Up @@ -536,9 +538,11 @@ func (tree *MutableTree) enableFastStorageAndCommitIfNotEnabled() (bool, error)
return false, err
}
}
fastItr.Close()
}

// Force garbage collection before we proceed to enabling fast storage.
runtime.GC()

if err := tree.enableFastStorageAndCommit(); err != nil {
tree.ndb.storageVersion = defaultStorageVersionValue
return false, err
Expand All @@ -555,10 +559,45 @@ func (tree *MutableTree) enableFastStorageAndCommitLocked() error {
func (tree *MutableTree) enableFastStorageAndCommit() error {
var err error

// We start a new thread to keep on checking if we are above 4GB, and if so garbage collect.
// This thread only lasts during the fast node migration.
// This is done to keep RAM usage down.
done := make(chan struct{})
defer func() {
done <- struct{}{}
close(done)
}()

go func () {
timer := time.NewTimer(time.Second)
var m runtime.MemStats

for {
// Sample the current memory usage
runtime.ReadMemStats(&m)

if m.Alloc > 4 * 1024 * 1024 * 1024 {
// If we are using more than 4GB of memory, we should trigger garbage collection
// to free up some memory.
runtime.GC()
}

select {
case <-timer.C:
timer.Reset(time.Second)
case <-done:
if !timer.Stop() {
<-timer.C
}
return
}
}
}()

itr := NewIterator(nil, nil, true, tree.ImmutableTree)
defer itr.Close()
for ; itr.Valid(); itr.Next() {
if err = tree.ndb.SaveFastNode(NewFastNode(itr.Key(), itr.Value(), tree.version)); err != nil {
if err = tree.ndb.SaveFastNodeNoCache(NewFastNode(itr.Key(), itr.Value(), tree.version)); err != nil {
return err
}
}
Expand Down
17 changes: 13 additions & 4 deletions nodedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,11 +216,18 @@ func (ndb *nodeDB) SaveNode(node *Node) {
ndb.cacheNode(node)
}

// SaveNode saves a FastNode to disk.
// SaveNode saves a FastNode to disk and add to cache.
func (ndb *nodeDB) SaveFastNode(node *FastNode) error {
ndb.mtx.Lock()
defer ndb.mtx.Unlock()
return ndb.saveFastNodeUnlocked(node)
return ndb.saveFastNodeUnlocked(node, true)
}

// SaveNode saves a FastNode to disk without adding to cache.
func (ndb *nodeDB) SaveFastNodeNoCache(node *FastNode) error {
ndb.mtx.Lock()
defer ndb.mtx.Unlock()
return ndb.saveFastNodeUnlocked(node, false)
}

// setFastStorageVersionToBatch sets storage version to fast where the version is
Expand Down Expand Up @@ -275,7 +282,7 @@ func (ndb *nodeDB) shouldForceFastStorageUpgrade() bool {
}

// SaveNode saves a FastNode to disk.
func (ndb *nodeDB) saveFastNodeUnlocked(node *FastNode) error {
func (ndb *nodeDB) saveFastNodeUnlocked(node *FastNode, shouldAddToCache bool) error {
if node.key == nil {
return fmt.Errorf("cannot have FastNode with a nil value for key")
}
Expand All @@ -291,7 +298,9 @@ func (ndb *nodeDB) saveFastNodeUnlocked(node *FastNode) error {
if err := ndb.batch.Set(ndb.fastNodeKey(node.key), buf.Bytes()); err != nil {
return fmt.Errorf("error while writing key/val to nodedb batch. Err: %w", err)
}
ndb.cacheFastNode(node)
if shouldAddToCache {
ndb.cacheFastNode(node)
}
return nil
}

Expand Down

0 comments on commit e44a79a

Please sign in to comment.