Merge pull request #4967 from aduffeck/posixfs-flakiness

[tests-only] Fix posixfs flakiness

.drone.star

@@ -761,7 +761,7 @@ def posixfsIntegrationTests(parallelRuns, skipExceptParts = []):
                         "environment": {
                             "TEST_SERVER_URL": "http://revad-services:20080",
                             "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
-                            "DELETE_USER_DATA_CMD": "bash -cx 'for i in {1..30}; do rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/* && break || sleep 5; done; sleep 1'",
+                            "DELETE_USER_DATA_CMD": "bash -cx 'rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/*'",
                             "STORAGE_DRIVER": "ocis",
                             "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
                             "TEST_WITH_LDAP": "true",

pkg/storage/fs/ocis/ocis.go

@@ -36,7 +36,7 @@ func init() {
 
 // New returns an implementation to of the storage.FS interface that talk to
 // a local filesystem.
-func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (storage.FS, error) {
+func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) {
 	o, err := options.New(m)
 	if err != nil {
 		return nil, err
@@ -47,5 +47,5 @@ func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (sto
 		return nil, err
 	}
 
-	return decomposedfs.NewDefault(m, bs, stream)
+	return decomposedfs.NewDefault(m, bs, stream, log)
 }

pkg/storage/fs/ocis/ocis_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cs3org/reva/v2/pkg/storage/fs/ocis"
 	"github.com/cs3org/reva/v2/tests/helpers"
+	"github.com/rs/zerolog"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -53,7 +54,7 @@ var _ = Describe("Ocis", func() {
 
 	Describe("New", func() {
 		It("returns a new instance", func() {
-			_, err := ocis.New(options, nil, nil)
+			_, err := ocis.New(options, nil, &zerolog.Logger{})
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

pkg/storage/fs/posix/posix.go

@@ -134,7 +134,7 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s
 		Trashbin:          trashbin,
 	}
 
-	dfs, err := decomposedfs.New(&o.Options, aspects)
+	dfs, err := decomposedfs.New(&o.Options, aspects, log)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/fs/posix/posix_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cs3org/reva/v2/pkg/storage/fs/posix"
 	"github.com/cs3org/reva/v2/tests/helpers"
+	"github.com/rs/zerolog"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -56,7 +57,7 @@ var _ = Describe("Posix", func() {
 
 	Describe("New", func() {
 		It("returns a new instance", func() {
-			_, err := posix.New(options, nil, nil)
+			_, err := posix.New(options, nil, &zerolog.Logger{})
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

pkg/storage/fs/posix/testhelpers/helpers.go

@@ -195,7 +195,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
 		Permissions: permissions.NewPermissions(pmock, permissionsSelector),
 		Trashbin:    tb,
 	}
-	fs, err := decomposedfs.New(&o.Options, aspects)
+	fs, err := decomposedfs.New(&o.Options, aspects, &logger)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/fs/s3ng/s3ng.go

@@ -35,7 +35,7 @@ func init() {
 
 // New returns an implementation to of the storage.FS interface that talk to
 // a local filesystem.
-func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (storage.FS, error) {
+func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) {
 	o, err := parseConfig(m)
 	if err != nil {
 		return nil, err
@@ -59,5 +59,5 @@ func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (sto
 		return nil, err
 	}
 
-	return decomposedfs.NewDefault(m, bs, stream)
+	return decomposedfs.NewDefault(m, bs, stream, log)
 }

pkg/storage/fs/s3ng/s3ng_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cs3org/reva/v2/pkg/storage/fs/s3ng"
 	"github.com/cs3org/reva/v2/tests/helpers"
+	"github.com/rs/zerolog"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -58,12 +59,12 @@ var _ = Describe("S3ng", func() {
 
 	Describe("New", func() {
 		It("fails on missing s3 configuration", func() {
-			_, err := s3ng.New(map[string]interface{}{}, nil, nil)
+			_, err := s3ng.New(map[string]interface{}{}, nil, &zerolog.Logger{})
 			Expect(err).To(MatchError("S3 configuration incomplete"))
 		})
 
 		It("works with complete configuration", func() {
-			_, err := s3ng.New(options, nil, nil)
+			_, err := s3ng.New(options, nil, &zerolog.Logger{})
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

pkg/storage/utils/decomposedfs/decomposedfs.go

@@ -35,6 +35,7 @@ import (
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	"github.com/jellydator/ttlcache/v2"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	tusd "github.com/tus/tusd/v2/pkg/handler"
 	microstore "go-micro.dev/v4/store"
 	"go.opentelemetry.io/otel"
@@ -125,10 +126,12 @@ type Decomposedfs struct {
 	userSpaceIndex  *spaceidindex.Index
 	groupSpaceIndex *spaceidindex.Index
 	spaceTypeIndex  *spaceidindex.Index
+
+	log *zerolog.Logger
 }
 
 // NewDefault returns an instance with default components
-func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) (storage.FS, error) {
+func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream, log *zerolog.Logger) (storage.FS, error) {
 	o, err := options.New(m)
 	if err != nil {
 		return nil, err
@@ -169,14 +172,12 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) (
 		Trashbin:          &DecomposedfsTrashbin{},
 	}
 
-	return New(o, aspects)
+	return New(o, aspects, log)
 }
 
 // New returns an implementation of the storage.FS interface that talks to
 // a local filesystem.
-func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) {
-	log := logger.New()
-
+func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (storage.FS, error) {
 	err := aspects.Tree.Setup()
 	if err != nil {
 		log.Error().Err(err).Msg("could not setup tree")
@@ -235,6 +236,7 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) {
 		userSpaceIndex:  userSpaceIndex,
 		groupSpaceIndex: groupSpaceIndex,
 		spaceTypeIndex:  spaceTypeIndex,
+		log:             log,
 	}
 	fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens)
 	if err = fs.trashbin.Setup(fs); err != nil {

pkg/storage/utils/decomposedfs/decomposedfs_test.go

@@ -61,7 +61,7 @@ var _ = Describe("Decomposed", func() {
 			_, err := decomposedfs.NewDefault(map[string]interface{}{
 				"root":           env.Root,
 				"permissionssvc": "any",
-			}, bs, nil)
+			}, bs, nil, nil)
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

pkg/storage/utils/decomposedfs/testhelpers/helpers.go

@@ -34,6 +34,7 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"github.com/cs3org/reva/v2/pkg/store"
 	"github.com/google/uuid"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/mock"
 	"google.golang.org/grpc"
 
@@ -178,7 +179,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
 		Permissions: permissions.NewPermissions(pmock, permissionsSelector),
 		Trashbin:    &decomposedfs.DecomposedfsTrashbin{},
 	}
-	fs, err := decomposedfs.New(o, aspects)
+	fs, err := decomposedfs.New(o, aspects, &zerolog.Logger{})
 	if err != nil {
 		return nil, err
 	}

pkg/storage/utils/decomposedfs/upload_async_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/cs3org/reva/v2/pkg/store"
 	"github.com/cs3org/reva/v2/pkg/utils"
 	"github.com/cs3org/reva/v2/tests/helpers"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/mock"
 	"google.golang.org/grpc"
 
@@ -180,7 +181,7 @@ var _ = Describe("Async file uploads", Ordered, func() {
 			EventStream: stream.Chan{pub, con},
 			Trashbin:    &DecomposedfsTrashbin{},
 		}
-		fs, err = New(o, aspects)
+		fs, err = New(o, aspects, &zerolog.Logger{})
 		Expect(err).ToNot(HaveOccurred())
 
 		resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"})

pkg/storage/utils/decomposedfs/upload_test.go

@@ -48,6 +48,7 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"github.com/cs3org/reva/v2/pkg/store"
 	"github.com/cs3org/reva/v2/tests/helpers"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/mock"
 	"google.golang.org/grpc"
 
@@ -144,7 +145,7 @@ var _ = Describe("File uploads", func() {
 			Permissions: permissions.NewPermissions(pmock, permissionsSelector),
 			Trashbin:    &decomposedfs.DecomposedfsTrashbin{},
 		}
-		fs, err = decomposedfs.New(o, aspects)
+		fs, err = decomposedfs.New(o, aspects, &zerolog.Logger{})
 		Expect(err).ToNot(HaveOccurred())
 
 		resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"})

tests/oc-integration-tests/drone/storage-users-posixfs.toml

@@ -24,7 +24,7 @@ root = "/drone/src/tmp/reva/data"
 permissionssvc = "localhost:10000"
 treetime_accounting = true
 treesize_accounting = true
-personalspacepath_template = "users/{{.User.Username}}"
+personalspacepath_template = "users/{{.User.Id.OpaqueId}}"
 generalspacepath_template = "projects/{{.SpaceId}}"
 watch_fs = true