Skip to content

Commit

Permalink
feat(chore): internal logger and WASM support
Browse files Browse the repository at this point in the history
  • Loading branch information
darkweak committed Jul 27, 2024
1 parent 48df68a commit 441d32f
Show file tree
Hide file tree
Showing 39 changed files with 370 additions and 187 deletions.
8 changes: 5 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
.PHONY: bump-version dependencies generate-release golangci-lint unit-tests

STORAGES_LIST=badger core etcd go-redis nats nuts olric otter redis
MODULES_LIST=badger core etcd go-redis nats nuts olric otter redis
STORAGES_LIST=badger etcd go-redis nats nuts olric otter redis
TESTS_LIST=badger core etcd go-redis nats nuts otter redis

bump-version:
Expand All @@ -17,21 +18,22 @@ bump-version:
sed -i '' 's/github.com\/darkweak\/storages\/otter $(from)/github.com\/darkweak\/storages\/otter $(to)/' otter/caddy/go.mod
sed -i '' 's/github.com\/darkweak\/storages\/redis $(from)/github.com\/darkweak\/storages\/redis $(to)/' redis/caddy/go.mod

for storage in $(STORAGES_LIST) ; do \
for storage in $(MODULES_LIST) ; do \
sed -i '' 's/github.com\/darkweak\/storages\/core $(from)/github.com\/darkweak\/storages\/core $(to)/' $$storage/go.mod ; \
sed -i '' 's/github.com\/darkweak\/storages\/core $(from)/github.com\/darkweak\/storages\/core $(to)/' $$storage/caddy/go.mod ; \
done

dependencies:
for storage in $(STORAGES_LIST) ; do \
cd $$storage && go mod tidy ; cd - ; \
cd $$storage/caddy && go mod tidy ; cd - ; \
done

generate-release:
cd .github/workflows && ./generate_release.sh

golangci-lint:
for storage in $(STORAGES_LIST) ; do \
for storage in $(MODULES_LIST) ; do \
cd $$storage && golangci-lint run --fix ; cd - ; \
done

Expand Down
30 changes: 18 additions & 12 deletions badger/badger.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
//go:build !wasm && !wasi

package badger

import (
Expand All @@ -22,7 +24,7 @@ import (
type Badger struct {
*badger.DB
stale time.Duration
logger *zap.Logger
logger core.Logger
}

var (
Expand All @@ -39,7 +41,7 @@ func (b *badgerLogger) Warningf(msg string, params ...interface{}) {
}

// Factory function create new Badger instance.
func Factory(badgerConfiguration core.CacheProvider, logger *zap.Logger, stale time.Duration) (core.Storer, error) {
func Factory(badgerConfiguration core.CacheProvider, logger core.Logger, stale time.Duration) (core.Storer, error) {
badgerOptions := badger.DefaultOptions(badgerConfiguration.Path)
badgerOptions.SyncWrites = true
badgerOptions.MemTableSize = 64 << 22
Expand All @@ -48,12 +50,12 @@ func Factory(badgerConfiguration core.CacheProvider, logger *zap.Logger, stale t
var parsedBadger badger.Options
if b, e := json.Marshal(badgerConfiguration.Configuration); e == nil {
if e = json.Unmarshal(b, &parsedBadger); e != nil {
logger.Sugar().Error("Impossible to parse the configuration for the default provider (Badger)", e)
logger.Error("Impossible to parse the configuration for the default provider (Badger)", e)
}
}

if err := mergo.Merge(&badgerOptions, parsedBadger, mergo.WithOverride); err != nil {
logger.Sugar().Error("An error occurred during the badgerOptions merge from the default options with your configuration.")
logger.Error("An error occurred during the badgerOptions merge from the default options with your configuration.")
}

if badgerOptions.InMemory {
Expand All @@ -72,7 +74,11 @@ func Factory(badgerConfiguration core.CacheProvider, logger *zap.Logger, stale t
badgerOptions = badgerOptions.WithInMemory(true)
}

badgerOptions.Logger = &badgerLogger{SugaredLogger: logger.Sugar()}
zapLogger, ok := logger.(*zap.SugaredLogger)
if ok {
badgerOptions.Logger = &badgerLogger{SugaredLogger: zapLogger}
}

uid := badgerOptions.Dir + badgerOptions.ValueDir + stale.String()

if instance, ok := enabledBadgerInstances.Load(uid); ok {
Expand All @@ -81,7 +87,7 @@ func Factory(badgerConfiguration core.CacheProvider, logger *zap.Logger, stale t

db, e := badger.Open(badgerOptions)
if e != nil {
logger.Sugar().Error("Impossible to open the Badger DB.", e)
logger.Error("Impossible to open the Badger DB.", e)
}

i := &Badger{DB: db, logger: logger, stale: stale}
Expand Down Expand Up @@ -228,14 +234,14 @@ func (provider *Badger) SetMultiLevel(baseKey, variedKey string, value []byte, v

compressed := new(bytes.Buffer)
if _, err = lz4.NewWriter(compressed).ReadFrom(bytes.NewReader(value)); err != nil {
provider.logger.Sugar().Errorf("Impossible to compress the key %s into Badger, %v", variedKey, err)
provider.logger.Errorf("Impossible to compress the key %s into Badger, %v", variedKey, err)

return err
}

err = btx.SetEntry(badger.NewEntry([]byte(variedKey), compressed.Bytes()).WithTTL(duration + provider.stale))
if err != nil {
provider.logger.Sugar().Errorf("Impossible to set the key %s into Badger, %v", variedKey, err)
provider.logger.Errorf("Impossible to set the key %s into Badger, %v", variedKey, err)

return err
}
Expand All @@ -244,7 +250,7 @@ func (provider *Badger) SetMultiLevel(baseKey, variedKey string, value []byte, v
item, err := btx.Get([]byte(mappingKey))

if err != nil && !errors.Is(err, badger.ErrKeyNotFound) {
provider.logger.Sugar().Errorf("Impossible to get the base key %s in Badger, %v", mappingKey, err)
provider.logger.Errorf("Impossible to get the base key %s in Badger, %v", mappingKey, err)

return err
}
Expand All @@ -264,12 +270,12 @@ func (provider *Badger) SetMultiLevel(baseKey, variedKey string, value []byte, v
return err
}

provider.logger.Sugar().Debugf("Store the new mapping for the key %s in Badger", variedKey)
provider.logger.Debugf("Store the new mapping for the key %s in Badger", variedKey)

return btx.SetEntry(badger.NewEntry([]byte(mappingKey), val))
})
if err != nil {
provider.logger.Sugar().Errorf("Impossible to set value into Badger, %v", err)
provider.logger.Errorf("Impossible to set value into Badger, %v", err)
}

return err
Expand All @@ -281,7 +287,7 @@ func (provider *Badger) Set(key string, value []byte, duration time.Duration) er
return txn.SetEntry(badger.NewEntry([]byte(key), value).WithTTL(duration))
})
if err != nil {
provider.logger.Sugar().Errorf("Impossible to set value into Badger, %v", err)
provider.logger.Errorf("Impossible to set value into Badger, %v", err)
}

return err
Expand Down
4 changes: 3 additions & 1 deletion badger/badger_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
//go:build !wasm && !wasi

package badger_test

import (
Expand All @@ -16,7 +18,7 @@ const (
)

func getBadgerInstance() (core.Storer, error) {
return badger.Factory(core.CacheProvider{}, zap.NewNop(), 0)
return badger.Factory(core.CacheProvider{}, zap.NewNop().Sugar(), 0)
}

// This test ensure that Badger options are override by the Souin configuration.
Expand Down
4 changes: 3 additions & 1 deletion badger/caddy/badger.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
//go:build !wasm && !wasi

package caddy

import (
Expand Down Expand Up @@ -109,7 +111,7 @@ func (Badger) CaddyModule() caddy.ModuleInfo {
// Provision to do the provisioning part.
func (b *Badger) Provision(ctx caddy.Context) error {
logger := ctx.Logger(b)
storer, err := badger.Factory(b.Configuration.Provider, logger, b.Configuration.Stale)
storer, err := badger.Factory(b.Configuration.Provider, logger.Sugar(), b.Configuration.Stale)
if err != nil {
return err
}
Expand Down
2 changes: 0 additions & 2 deletions badger/caddy/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -670,8 +670,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down
3 changes: 1 addition & 2 deletions badger/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
21 changes: 11 additions & 10 deletions core/core.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
//go:build !wasi && !wasm

package core

import (
Expand All @@ -9,7 +11,6 @@ import (
"time"

lz4 "github.com/pierrec/lz4/v4"
"go.uber.org/zap"
)

type keyIndex struct {
Expand Down Expand Up @@ -60,7 +61,7 @@ func DecodeMapping(item []byte) (mapping StorageMapper, e error) {
return
}

func MappingElection(provider Storer, item []byte, req *http.Request, validator *Revalidator, logger *zap.Logger) (resultFresh *http.Response, resultStale *http.Response, e error) {
func MappingElection(provider Storer, item []byte, req *http.Request, validator *Revalidator, logger Logger) (resultFresh *http.Response, resultStale *http.Response, e error) {
var mapping StorageMapper

if len(item) != 0 {
Expand Down Expand Up @@ -97,12 +98,12 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
_, _ = reader.WriteTo(bufW)

if resultFresh, e = http.ReadResponse(bufio.NewReader(bufW), req); e != nil {
logger.Sugar().Errorf("An error occurred while reading response for the key %s: %v", keyName, e)
logger.Errorf("An error occurred while reading response for the key %s: %v", keyName, e)

return resultFresh, resultStale, e
}

logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", keyName, validator)
logger.Debugf("The stored key %s matched the current iteration key ETag %+v", keyName, validator)

return resultFresh, resultStale, e
}
Expand All @@ -117,30 +118,30 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
_, _ = reader.WriteTo(bufW)

if resultStale, e = http.ReadResponse(bufio.NewReader(bufW), req); e != nil {
logger.Sugar().Errorf("An error occurred while reading response for the key %s: %v", keyName, e)
logger.Errorf("An error occurred while reading response for the key %s: %v", keyName, e)

return resultFresh, resultStale, e
}

logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v as stale", keyName, validator)
logger.Debugf("The stored key %s matched the current iteration key ETag %+v as stale", keyName, validator)
}
}
} else {
logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", keyName, validator)
logger.Debugf("The stored key %s didn't match the current iteration key ETag %+v", keyName, validator)
}
}

return resultFresh, resultStale, e
}

func MappingUpdater(key string, item []byte, logger *zap.Logger, now, freshTime, staleTime time.Time, variedHeaders http.Header, etag, realKey string) (val []byte, e error) {
func MappingUpdater(key string, item []byte, logger Logger, now, freshTime, staleTime time.Time, variedHeaders http.Header, etag, realKey string) (val []byte, e error) {
var mapping StorageMapper
if len(item) == 0 {
mapping = StorageMapper{}
} else {
e = gob.NewDecoder(bytes.NewBuffer(item)).Decode(&mapping)
if e != nil {
logger.Sugar().Errorf("Impossible to decode the key %s, %v", key, e)
logger.Errorf("Impossible to decode the key %s, %v", key, e)

return nil, e
}
Expand All @@ -163,7 +164,7 @@ func MappingUpdater(key string, item []byte, logger *zap.Logger, now, freshTime,

e = gob.NewEncoder(buf).Encode(mapping)
if e != nil {
logger.Sugar().Errorf("Impossible to encode the mapping value for the key %s, %v", key, e)
logger.Errorf("Impossible to encode the mapping value for the key %s, %v", key, e)

return nil, e
}
Expand Down
Loading

0 comments on commit 441d32f

Please sign in to comment.