diff --git a/.gitignore b/.gitignore index 789e187529..1c338734c7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ # Ignore builds cmd/revad/revad +cmd/revad/revad-ceph cmd/revad/revad.pid cmd/reva/reva cmd/revad/main/main diff --git a/CODEOWNERS b/CODEOWNERS index e6ee4e0407..f5048e7f2f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @labkode @glpatcern @diocas @jessejeens +* @glpatcern @diocas @jessegeens diff --git a/Makefile b/Makefile index e8d9e63fe2..d5baea91bb 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,11 @@ gaia: .PHONY: cernbox-revad cernbox-revad: gaia - gaia build --with github.com/cernbox/reva-plugins --with github.com/cs3org/reva=$(shell pwd) -o ./cmd/revad/revad + gaia build --with github.com/cernbox/reva-plugins --with github.com/cs3org/reva=$(shell pwd) --debug -o ./cmd/revad/revad + +.PHONY: cernbox-revad-ceph +cernbox-revad-ceph: cernbox-revad + gaia build --tags ceph --with github.com/cernbox/reva-plugins --with github.com/cs3org/reva=$(shell pwd) --debug -o ./cmd/revad/revad-ceph .PHONY: revad-ceph revad-ceph: diff --git a/README.md b/README.md index 443f9d8ae1..0b78987f93 100644 --- a/README.md +++ b/README.md @@ -83,4 +83,9 @@ Reva is distributed under [Apache 2.0 license](https://github.com/cs3org/reva/bl ## Logo -Reva logo's have been designed and contributed to the project by Eamon Maguire. +Reva logo's have been designed and contributed to the project by Eamonn Maguire. + +## History +This project was initially conceived and brought to life by Hugo Gonzalez Labrador (@labkode) in 2017. +Since its roots, Reva has evolved and expanded thanks to the passion and commitment of +dozens of remarkable individual contributors. [Learn more](https://reva.link/about/). diff --git a/changelog/unreleased/ceph-refactoring.md b/changelog/unreleased/ceph-refactoring.md new file mode 100644 index 0000000000..94f1823977 --- /dev/null +++ b/changelog/unreleased/ceph-refactoring.md @@ -0,0 +1,3 @@ +Enhancement: cephfs refactoring + make home layout configurable + +https://github.com/cs3org/reva/pull/4911 diff --git a/changelog/unreleased/change-project-ownership.md b/changelog/unreleased/change-project-ownership.md new file mode 100644 index 0000000000..911b56b3f3 --- /dev/null +++ b/changelog/unreleased/change-project-ownership.md @@ -0,0 +1,5 @@ +Enhancement: @labkode steps down as project owner + +Hugo (@labkode) steps down as project owner of Reva. + +https://github.com/cs3org/reva/pull/4937 diff --git a/changelog/unreleased/enhance-ceph.md b/changelog/unreleased/enhance-ceph.md new file mode 100644 index 0000000000..b4fc855ff7 --- /dev/null +++ b/changelog/unreleased/enhance-ceph.md @@ -0,0 +1,3 @@ +Enhancement: Refactor Ceph code + +https://github.com/cs3org/reva/pull/4824 diff --git a/changelog/unreleased/eos-grpc-token-access.md b/changelog/unreleased/eos-grpc-token-access.md new file mode 100644 index 0000000000..7842c4b83c --- /dev/null +++ b/changelog/unreleased/eos-grpc-token-access.md @@ -0,0 +1,6 @@ +Enhancement: access to EOS via tokens over gRPC + +As a guest account, accessing a file shared with you relies on a token that is generated on behalf of the resource owner. This method, GenerateToken, has now been implemented in the EOS gRPC client. Additionally, the HTTP client now takes tokens into account. + + +https://github.com/cs3org/reva/pull/4934 \ No newline at end of file diff --git a/changelog/unreleased/fix-eosgrpc-stat.md b/changelog/unreleased/fix-eosgrpc-stat.md new file mode 100644 index 0000000000..3970628591 --- /dev/null +++ b/changelog/unreleased/fix-eosgrpc-stat.md @@ -0,0 +1,5 @@ +Bugfix: fixed tree metadata following fix in EOS + +The treecount is now populated from the EOS response. + +https://github.com/cs3org/reva/pull/4931 diff --git a/changelog/unreleased/fix-fav-grpc.md b/changelog/unreleased/fix-fav-grpc.md new file mode 100644 index 0000000000..e5d5645b35 --- /dev/null +++ b/changelog/unreleased/fix-fav-grpc.md @@ -0,0 +1,5 @@ +Bugfix: make removal of favourites work + +Currently, removing a folder from your favourites is broken, because the handleFavAttr method is only called in SetAttr, not in UnsetAttr. This change fixes this. + +https://github.com/cs3org/reva/pull/4930 \ No newline at end of file diff --git a/changelog/unreleased/log-home.md b/changelog/unreleased/log-home.md new file mode 100644 index 0000000000..4a82170ab1 --- /dev/null +++ b/changelog/unreleased/log-home.md @@ -0,0 +1,3 @@ +Enhancement: improved logging on createHome + +https://github.com/cs3org/reva/pull/4970 diff --git a/changelog/unreleased/parse-favs-grpc.md b/changelog/unreleased/parse-favs-grpc.md new file mode 100644 index 0000000000..65bd1da412 --- /dev/null +++ b/changelog/unreleased/parse-favs-grpc.md @@ -0,0 +1,5 @@ +Bugfix: handle parsing of favs over gRPC + +To store user favorites, the key `user.http://owncloud.org/ns/favorite` maps to a list of users, in the format `u:username=1`. Right now, extracting the "correct" user doesn't happen in gRPC, while it is implemented in the EOS binary client. This feature has now been moved to the higher-level call in eosfs. + +https://github.com/cs3org/reva/pull/4973 \ No newline at end of file diff --git a/changelog/unreleased/publink-fix.md b/changelog/unreleased/publink-fix.md new file mode 100644 index 0000000000..6d626bd4fc --- /dev/null +++ b/changelog/unreleased/publink-fix.md @@ -0,0 +1,3 @@ +Bugfix: public links: return error when owner could not be resolved + +https://github.com/cs3org/reva/pull/4907 diff --git a/changelog/unreleased/undo-gw-refactoring.md b/changelog/unreleased/undo-gw-refactoring.md new file mode 100644 index 0000000000..5d4bf8c022 --- /dev/null +++ b/changelog/unreleased/undo-gw-refactoring.md @@ -0,0 +1,7 @@ +Bugfix: revert 'make home layout configurable' + +Partial revert of #4911, to be re-added after +more testing and configuration validation. The +eoshome vs eos storage drivers are to be adapted. + +https://github.com/cs3org/reva/pull/4939 diff --git a/go.mod b/go.mod index 4d37a9bccf..d20e7d1a7d 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module github.com/cs3org/reva require ( - bou.ke/monkey v1.0.2 github.com/BurntSushi/toml v1.3.2 github.com/CiscoM31/godata v1.0.8 github.com/Masterminds/sprig v2.22.0+incompatible @@ -32,15 +31,15 @@ require ( github.com/gomodule/redigo v1.9.2 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 - github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/jt-nti/gproto v0.0.0-20210304092907-23e645af1351 github.com/juliangruber/go-intersect v1.1.0 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.24 github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.3.4 github.com/mitchellh/mapstructure v1.5.0 - github.com/nats-io/nats.go v1.33.1 + github.com/nats-io/nats.go v1.37.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.30.0 github.com/owncloud/libre-graph-api-go v1.0.5-0.20240425090020-dba6d1507c38 @@ -59,7 +58,7 @@ require ( golang.org/x/crypto v0.23.0 golang.org/x/oauth2 v0.20.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.20.0 + golang.org/x/sys v0.27.0 golang.org/x/term v0.20.0 google.golang.org/genproto v0.0.0-20240314234333-6e1732d8331c google.golang.org/grpc v1.65.0 @@ -94,6 +93,7 @@ require ( github.com/gocraft/dbr/v2 v2.7.2 // indirect github.com/golang/glog v1.2.1 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/huandu/xstrings v1.4.0 // indirect diff --git a/go.sum b/go.sum index 6f8f472973..2d392c9a69 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= -bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -1165,8 +1163,8 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 h1:f4tggROQKKcnh4eItay6z/HbHLqghBxS8g7pyMhmDio= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0/go.mod h1:hKAkSgNkL0FII46ZkJcpVEAai4KV+swlIWCKfekd1pA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -1301,8 +1299,8 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/mattn/go-tty v0.0.5 h1:s09uXI7yDbXzzTTfw3zonKFzwGkyYlgU3OMjqA0ddz4= github.com/mattn/go-tty v0.0.5/go.mod h1:u5GGXBtZU6RQoKV8gY5W6UhMudbR5vXnUe7j3pxse28= @@ -1342,8 +1340,8 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= -github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE= +github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= @@ -1910,8 +1908,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/grpc/services/gateway/authprovider.go b/internal/grpc/services/gateway/authprovider.go index 85aa3f29b7..2492e4d28d 100644 --- a/internal/grpc/services/gateway/authprovider.go +++ b/internal/grpc/services/gateway/authprovider.go @@ -163,7 +163,7 @@ func (s *svc) Authenticate(ctx context.Context, req *gateway.AuthenticateRequest if createHomeRes.Status.Code != rpc.Code_CODE_OK { err := status.NewErrorFromCode(createHomeRes.Status.Code, "gateway") - log.Err(err).Msg("error calling Createhome") + log.Err(err).Any("response", createHomeRes).Msg("return from CreateHome") return &gateway.AuthenticateResponse{ Status: status.NewInternal(ctx, err, "error creating user home"), }, nil diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index a9278a13c9..9a9d444ea2 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -84,6 +84,7 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( home := s.getHome(ctx) c, err := s.findByPath(ctx, home) if err != nil { + log.Err(err).Str("home", home).Msg("gateway: error finding home on storage provider") return &provider.CreateHomeResponse{ Status: status.NewStatusFromErrType(ctx, "error finding home", err), }, nil @@ -91,7 +92,7 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( res, err := c.CreateHome(ctx, req) if err != nil { - log.Err(err).Msg("gateway: error creating home on storage provider") + log.Err(err).Str("home", home).Msg("gateway: error creating home on storage provider") return &provider.CreateHomeResponse{ Status: status.NewInternal(ctx, err, "error calling CreateHome"), }, nil diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 605cd5cc88..7f4029d3ef 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -322,7 +322,6 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { return } httpReq.Header = r.Header - httpRes, err := httpClient.Do(httpReq) if err != nil { log.Err(err).Msg("error doing PUT request to data service") diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index d87e257fa7..4a0930281e 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -261,6 +261,7 @@ func (s *svc) getResourceInfos(ctx context.Context, w http.ResponseWriter, r *ht Ref: ref, ArbitraryMetadataKeys: metadataKeys, } + res, err := client.Stat(ctx, req) if err != nil { log.Error().Err(err).Interface("req", req).Msg("error sending a grpc stat request") diff --git a/internal/http/services/owncloud/ocdav/proppatch.go b/internal/http/services/owncloud/ocdav/proppatch.go index 82d41a290f..e72ffd10c1 100644 --- a/internal/http/services/owncloud/ocdav/proppatch.go +++ b/internal/http/services/owncloud/ocdav/proppatch.go @@ -228,7 +228,7 @@ func (s *svc) handleProppatch(ctx context.Context, w http.ResponseWriter, r *htt HandleErrorStatus(&log, w, res.Status) return nil, nil, false } - if key == "http://owncloud.org/ns/favorite" { + if key == _propOcFavorite { statRes, err := c.Stat(ctx, &provider.StatRequest{Ref: ref}) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -269,7 +269,7 @@ func (s *svc) handleProppatch(ctx context.Context, w http.ResponseWriter, r *htt acceptedProps = append(acceptedProps, propNameXML) delete(sreq.ArbitraryMetadata.Metadata, key) - if key == "http://owncloud.org/ns/favorite" { + if key == _propOcFavorite { statRes, err := c.Stat(ctx, &provider.StatRequest{Ref: ref}) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/auth/manager/ldap/ldap.go b/pkg/auth/manager/ldap/ldap.go index 16374226e0..7bfe436564 100644 --- a/pkg/auth/manager/ldap/ldap.go +++ b/pkg/auth/manager/ldap/ldap.go @@ -124,7 +124,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) log := appctx.GetLogger(ctx) l, err := utils.GetLDAPConnection(&am.c.LDAPConn) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "error creating ldap connection") } defer l.Close() @@ -139,10 +139,10 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) sr, err := l.Search(searchRequest) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrapf(err, "error searching. seachrequest = %+v", searchRequest) } - log.Trace().Interface("entries", sr.Entries).Send() + log.Debug().Interface("entries", sr.Entries).Send() if len(sr.Entries) != 1 { return nil, nil, errtypes.NotFound(clientID) } @@ -153,7 +153,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) err = l.Bind(userdn, clientSecret) if err != nil { log.Debug().Err(err).Interface("userdn", userdn).Msg("bind with user credentials failed") - return nil, nil, err + return nil, nil, errors.Wrapf(err, "error binding with user credentials for user %s", userdn) } userID := &user.UserId{ @@ -193,7 +193,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) u := &user.User{ Id: userID, // TODO add more claims from the StandardClaims, eg EmailVerified - Username: sr.Entries[0].GetEqualFoldAttributeValue(am.c.Schema.CN), + Username: sr.Entries[0].GetEqualFoldAttributeValue(am.c.Schema.UID), // TODO groups Groups: getGroupsResp.Groups, Mail: sr.Entries[0].GetEqualFoldAttributeValue(am.c.Schema.Mail), diff --git a/pkg/auth/manager/publicshares/publicshares.go b/pkg/auth/manager/publicshares/publicshares.go index 3739999d98..05eabc47f1 100644 --- a/pkg/auth/manager/publicshares/publicshares.go +++ b/pkg/auth/manager/publicshares/publicshares.go @@ -128,6 +128,9 @@ func (m *manager) Authenticate(ctx context.Context, token, secret string) (*user if err != nil { return nil, nil, err } + if getUserResponse.Status.Code != rpcv1beta1.Code_CODE_OK { + return nil, nil, errtypes.NotFound(getUserResponse.Status.Message) + } share := publicShareResponse.GetShare() role := authpb.Role_ROLE_VIEWER diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 3f899b5e75..ff9b2d407d 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -379,7 +379,7 @@ func (c *Client) GetFileInfoByInode(ctx context.Context, auth eosclient.Authoriz if err != nil { return nil, err } - info, err := c.parseFileInfo(ctx, stdout, true) + info, err := c.parseFileInfo(ctx, stdout) if err != nil { return nil, err } @@ -403,7 +403,7 @@ func (c *Client) GetFileInfoByFXID(ctx context.Context, auth eosclient.Authoriza return nil, err } - info, err := c.parseFileInfo(ctx, stdout, true) + info, err := c.parseFileInfo(ctx, stdout) if err != nil { return nil, err } @@ -418,7 +418,7 @@ func (c *Client) GetFileInfoByPath(ctx context.Context, auth eosclient.Authoriza if err != nil { return nil, err } - info, err := c.parseFileInfo(ctx, stdout, true) + info, err := c.parseFileInfo(ctx, stdout) if err != nil { return nil, err } @@ -442,7 +442,7 @@ func (c *Client) getRawFileInfoByPath(ctx context.Context, auth eosclient.Author if err != nil { return nil, err } - return c.parseFileInfo(ctx, stdout, false) + return c.parseFileInfo(ctx, stdout) } func (c *Client) mergeACLsAndAttrsForFiles(ctx context.Context, auth eosclient.Authorization, info *eosclient.FileInfo) *eosclient.FileInfo { @@ -970,7 +970,7 @@ func (c *Client) parseFind(ctx context.Context, auth eosclient.Authorization, di if rl == "" { continue } - fi, err := c.parseFileInfo(ctx, rl, true) + fi, err := c.parseFileInfo(ctx, rl) if err != nil { return nil, err } @@ -1071,7 +1071,7 @@ func (c *Client) parseQuota(path, raw string) (*eosclient.QuotaInfo, error) { } // TODO(labkode): better API to access extended attributes. -func (c *Client) parseFileInfo(ctx context.Context, raw string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { +func (c *Client) parseFileInfo(ctx context.Context, raw string) (*eosclient.FileInfo, error) { line := raw[15:] index := strings.Index(line, " file=/") lengthString := line[0:index] @@ -1111,7 +1111,7 @@ func (c *Client) parseFileInfo(ctx context.Context, raw string, parseFavoriteKey } } } - fi, err := c.mapToFileInfo(ctx, kv, attrs, parseFavoriteKey) + fi, err := c.mapToFileInfo(ctx, kv, attrs) if err != nil { return nil, err } @@ -1121,7 +1121,7 @@ func (c *Client) parseFileInfo(ctx context.Context, raw string, parseFavoriteKey // mapToFileInfo converts the dictionary to an usable structure. // The kv has format: // map[sys.forced.space:default files:0 mode:42555 ino:5 sys.forced.blocksize:4k sys.forced.layout:replica uid:0 fid:5 sys.forced.blockchecksum:crc32c sys.recycle:/eos/backup/proc/recycle/ fxid:00000005 pid:1 etag:5:0.000 keylength.file:4 file:/eos treesize:1931593933849913 container:3 gid:0 mtime:1498571294.108614409 ctime:1460121992.294326762 pxid:00000001 sys.forced.checksum:adler sys.forced.nstripes:2]. -func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { +func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string) (*eosclient.FileInfo, error) { inode, err := strconv.ParseUint(kv["ino"], 10, 64) if err != nil { return nil, err @@ -1225,11 +1225,6 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, return nil, err } - // Read the favorite attr - if parseFavoriteKey { - parseAndSetFavoriteAttr(ctx, attrs) - } - fi := &eosclient.FileInfo{ File: kv["file"], Inode: inode, @@ -1255,26 +1250,3 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, return fi, nil } - -func parseAndSetFavoriteAttr(ctx context.Context, attrs map[string]string) { - // Read and correctly set the favorite attr - if user, ok := appctx.ContextGetUser(ctx); ok { - if favAttrStr, ok := attrs[favoritesKey]; ok { - favUsers, err := acl.Parse(favAttrStr, acl.ShortTextForm) - if err != nil { - return - } - for _, u := range favUsers.Entries { - // Check if the current user has favorited this resource - if u.Qualifier == user.Id.OpaqueId { - // Set attr val to 1 - attrs[favoritesKey] = "1" - return - } - } - } - } - - // Delete the favorite attr from the response - delete(attrs, favoritesKey) -} diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go index 22f46e590c..fb867c5a95 100644 --- a/pkg/eosclient/eosgrpc/eosgrpc.go +++ b/pkg/eosclient/eosgrpc/eosgrpc.go @@ -125,6 +125,10 @@ type Options struct { // SecProtocol is the comma separated list of security protocols used by xrootd. // For example: "sss, unix" SecProtocol string + + // TokenExpiry stores in seconds the time after which generated tokens will expire + // Default is 3600 + TokenExpiry int } func getUser(ctx context.Context) (*userpb.User, error) { @@ -602,6 +606,15 @@ func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, at log := appctx.GetLogger(ctx) log.Info().Str("func", "UnsetAttr").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") + // Favorites need to be stored per user so handle these separately + if attr.Type == eosclient.UserAttr && attr.Key == favoritesKey { + info, err := c.GetFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + return c.handleFavAttr(ctx, auth, attr, recursive, path, info, false) + } + // Initialize the common fields of the NSReq rq, err := c.initNSRequest(ctx, auth, app) if err != nil { @@ -1603,7 +1616,49 @@ func (c *Client) ReadVersion(ctx context.Context, auth eosclient.Authorization, // GenerateToken returns a token on behalf of the resource owner to be used by lightweight accounts. func (c *Client) GenerateToken(ctx context.Context, auth eosclient.Authorization, path string, a *acl.Entry) (string, error) { - return "", errtypes.NotSupported("TODO") + log := appctx.GetLogger(ctx) + log.Info().Str("func", "GenerateToken").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") + + // Initialize the common fields of the NSReq + rq, err := c.initNSRequest(ctx, auth, "") + if err != nil { + log.Error().Str("func", "GenerateToken").Str("err", err.Error()).Msg("Error on initNSRequest") + return "", err + } + + msg := new(erpc.NSRequest_TokenRequest) + msg.Token = &erpc.ShareToken{} + msg.Token.Token = &erpc.ShareProto{} + msg.Token.Token.Permission = a.Permissions + msg.Token.Token.Expires = uint64(time.Now().Add(time.Duration(c.opt.TokenExpiry) * time.Second).Unix()) + msg.Token.Token.Allowtree = true + msg.Token.Token.Path = path + + rq.Command = &erpc.NSRequest_Token{ + Token: msg, + } + + // Now send the req and see what happens + resp, err := c.cl.Exec(appctx.ContextGetClean(ctx), rq) + e := c.getRespError(resp, err) + if e != nil { + log.Error().Str("func", "GenerateToken").Str("err", e.Error()).Msg("") + return "", e + } + + if resp == nil { + log.Error().Str("func", "GenerateToken").Msg("nil grpc response") + return "", errtypes.InternalError(fmt.Sprintf("nil response for uid: '%s' ", auth.Role.UID)) + } + + // For some reason, the token is embedded in the error, with error code 0 + if resp.GetError() != nil { + if resp.GetError().Code == 0 { + return resp.GetError().Msg, nil + } + } + log.Error().Str("func", "GenerateToken").Msg("GenerateToken over gRPC expected an error but did not receive one") + return "", err } func (c *Client) getVersionFolderInode(ctx context.Context, auth eosclient.Authorization, p string) (uint64, error) { @@ -1677,8 +1732,7 @@ func (c *Client) grpcMDResponseToFileInfo(ctx context.Context, st *erpc.MDRespon fi.TreeSize = uint64(st.Cmd.TreeSize) fi.Size = fi.TreeSize - // TODO(lopresti) this info is missing in the EOS Protobuf, cf. EOS-5974 - // fi.TreeCount = uint64(st.Cmd.TreeCount) + fi.TreeCount = st.Cmd.Files + st.Cmd.Containers log.Debug().Str("stat info - path", fi.File).Uint64("inode", fi.Inode).Uint64("uid", fi.UID).Uint64("gid", fi.GID).Str("etag", fi.ETag).Msg("grpc response") } else { diff --git a/pkg/eosclient/eosgrpc/eoshttp.go b/pkg/eosclient/eosgrpc/eoshttp.go index e447afea13..bab34cf5b5 100644 --- a/pkg/eosclient/eosgrpc/eoshttp.go +++ b/pkg/eosclient/eosgrpc/eoshttp.go @@ -241,7 +241,9 @@ func (c *EOSHTTPClient) buildFullURL(urlpath string, auth eosclient.Authorizatio fullurl += "?" } - if auth.Role.UID != "" { + if auth.Token != "" { + fullurl += "authz=" + auth.Token + } else if auth.Role.UID != "" { fullurl += fmt.Sprintf("eos.ruid=%s&eos.rgid=%s", auth.Role.UID, auth.Role.GID) } @@ -291,7 +293,15 @@ func (c *EOSHTTPClient) GETFile(ctx context.Context, remoteuser string, auth eos // Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream log.Debug().Str("func", "GETFile").Str("finalurl", finalurl).Msg("sending req") - resp, err := c.doReq(req, remoteuser) + // c.doReq sets headers such as remoteuser and x-gateway-authorization + // we don't want those when using a token (i.e. ?authz=), so in this case + // we skip this and call the HTTP client directly + var resp *http.Response + if auth.Token != "" { + resp, err = c.cl.Do(req) + } else { + resp, err = c.doReq(req, remoteuser) + } // Let's support redirections... and if we retry we have to retry at the same FST, avoid going back to the MGM if resp != nil && (resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect) { @@ -390,7 +400,15 @@ func (c *EOSHTTPClient) PUTFile(ctx context.Context, remoteuser string, auth eos // Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream log.Debug().Str("func", "PUTFile").Msg("sending req") - resp, err := c.doReq(req, remoteuser) + // c.doReq sets headers such as remoteuser and x-gateway-authorization + // we don't want those when using a token (i.e. ?authz=), so in this case + // we skip this and call the HTTP client directly + var resp *http.Response + if auth.Token != "" { + resp, err = c.cl.Do(req) + } else { + resp, err = c.doReq(req, remoteuser) + } // Let's support redirections... and if we retry we retry at the same FST if resp != nil && resp.StatusCode == 307 { diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go index 1f32df6bb2..0efdeb069f 100644 --- a/pkg/storage/fs/cephfs/cephfs.go +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -48,14 +48,8 @@ import ( ) const ( - xattrTrustedNs = "trusted." - xattrEID = xattrTrustedNs + "eid" - xattrMd5 = xattrTrustedNs + "checksum" - xattrMd5ts = xattrTrustedNs + "checksumTS" - xattrRef = xattrTrustedNs + "ref" - xattrUserNs = "user." - snap = ".snap" - xattrLock = xattrUserNs + "reva.lockpayload" + xattrUserNs = "user." + xattrLock = xattrUserNs + "reva.lockpayload" ) type cephfs struct { @@ -69,8 +63,8 @@ func init() { registry.Register("cephfs", New) } -// New returns an implementation to of the storage.FS interface that talks to -// a ceph filesystem. +// New returns an implementation of the storage.FS interface that talks to +// a CephFS storage via libcephfs. func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err error) { var o Options if err := cfg.Decode(m, &o); err != nil { @@ -82,19 +76,9 @@ func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err erro return nil, errors.New("cephfs: can't create caches") } - adminConn := newAdminConn(&o) - if adminConn == nil { - return nil, errors.Wrap(err, "cephfs: Couldn't create admin connections") - } - - for _, dir := range []string{o.ShadowFolder, o.UploadFolder} { - _, err := adminConn.adminMount.Statx(dir, goceph.StatxMask(goceph.StatxIno), 0) - if err != nil { - err = adminConn.adminMount.MakeDir(dir, dirPermFull) - if err != nil && err.Error() != errFileExists { - return nil, errors.New("cephfs: can't initialise system dir " + dir + ":" + err.Error()) - } - } + adminConn, err := newAdminConn(&o) + if err != nil { + return nil, errors.Wrap(err, "cephfs: couldn't create admin connections") } return &cephfs{ @@ -105,70 +89,66 @@ func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err erro } func (fs *cephfs) GetHome(ctx context.Context) (string, error) { - if fs.conf.DisableHome { - return "", errtypes.NotSupported("cephfs: GetHome() home supported disabled") - } - + log := appctx.GetLogger(ctx) user := fs.makeUser(ctx) - + log.Debug().Interface("user", user).Msg("GetHome for user") return user.home, nil } func (fs *cephfs) CreateHome(ctx context.Context) (err error) { - if fs.conf.DisableHome { - return errtypes.NotSupported("cephfs: GetHome() home supported disabled") - } + log := appctx.GetLogger(ctx) user := fs.makeUser(ctx) + log.Debug().Interface("user", user).Msg("CreateHome for user") - // Stop createhome from running the whole thing because it is called multiple times - if _, err = fs.adminConn.adminMount.Statx(user.home, goceph.StatxMode, 0); err == nil { - return + // Skip home creation if the directory already exists. + // We do not check for all necessary attributes, only for the existence of the directory. + stat, err := fs.adminConn.adminMount.Statx(user.home, goceph.StatxMode, 0) + if err != nil { + return errors.Wrap(err, "error stating user home when trying to create it") } + log.Debug().Interface("stat", stat).Msgf("home is %s") + + // TODO(labkode): for now we always try to create the home directory even if it exists. + // One needs to check for "no such of file or directory" error to short-cut. + err = walkPath(user.home, func(path string) error { return fs.adminConn.adminMount.MakeDir(path, fs.conf.DirPerms) }, false) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } err = fs.adminConn.adminMount.Chown(user.home, uint32(user.UidNumber), uint32(user.GidNumber)) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } err = fs.adminConn.adminMount.SetXattr(user.home, "ceph.quota.max_bytes", []byte(fmt.Sprint(fs.conf.UserQuotaBytes)), 0) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } - user.op(func(cv *cacheVal) { - err = cv.mount.MakeDir(removeLeadingSlash(fs.conf.ShareFolder), fs.conf.DirPerms) - if err != nil && err.Error() == errFileExists { - err = nil - } - }) - - return getRevaError(err) + return nil } func (fs *cephfs) CreateDir(ctx context.Context, ref *provider.Reference) error { user := fs.makeUser(ctx) path, err := user.resolveRef(ref) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { if err = cv.mount.MakeDir(path, fs.conf.DirPerms); err != nil { + log.Debug().Str("path", path).Err(err).Msg("cv.mount.CreateDir returned") return } - - //TODO(tmourati): Add entry id logic }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { @@ -179,20 +159,22 @@ func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err erro return err } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { if err = cv.mount.Unlink(path); err != nil && err.Error() == errIsADirectory { err = cv.mount.RemoveDir(path) } - - //TODO(tmourati): Add entry id logic }) - //has already been deleted by direct mount - if err != nil && err.Error() == errNotFound { - return nil + if err != nil { + log.Debug().Any("ref", ref).Err(err).Msg("Delete returned") + if err.Error() == errNotFound { + //has already been deleted by direct mount + return nil + } } - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { @@ -205,12 +187,12 @@ func (fs *cephfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) return } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { if err = cv.mount.Rename(oldPath, newPath); err != nil { + log.Debug().Any("oldRef", oldRef).Any("newRef", newRef).Err(err).Msg("cv.mount.Rename returned") return } - - //TODO(tmourati): Add entry id logic, handle already moved file error }) // has already been moved by direct mount @@ -218,13 +200,17 @@ func (fs *cephfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) return nil } - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + if ref == nil { + return nil, errors.New("error: ref is nil") + } + + log := appctx.GetLogger(ctx) var path string user := fs.makeUser(ctx) - if path, err = user.resolveRef(ref); err != nil { return nil, err } @@ -232,37 +218,43 @@ func (fs *cephfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []s user.op(func(cv *cacheVal) { var stat Statx if stat, err = cv.mount.Statx(path, goceph.StatxBasicStats, 0); err != nil { + log.Debug().Str("path", path).Err(err).Msg("cv.mount.Statx returned") return } ri, err = user.fileAsResourceInfo(cv, path, stat, mdKeys) + if err != nil { + log.Debug().Any("resourceInfo", ri).Err(err).Msg("fileAsResourceInfo returned") + } }) - return ri, getRevaError(err) + return ri, getRevaError(ctx, err) } func (fs *cephfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (files []*provider.ResourceInfo, err error) { - var path string - user := fs.makeUser(ctx) - if path, err = user.resolveRef(ref); err != nil { - return + if ref == nil { + return nil, errors.New("error: ref is nil") } - // The user wants to access their home, create it if it doesn't exist - if path == fs.conf.Root { - if err = fs.CreateHome(ctx); err != nil { - return - } + log := appctx.GetLogger(ctx) + log.Debug().Interface("ref", ref) + user := fs.makeUser(ctx) + + var path string + if path, err = user.resolveRef(ref); err != nil { + return nil, err } user.op(func(cv *cacheVal) { var dir *goceph.Directory if dir, err = cv.mount.OpenDir(path); err != nil { + log.Debug().Str("path", path).Err(err).Msg("cv.mount.OpenDir returned") return } defer closeDir(dir) var entry *goceph.DirEntryPlus var ri *provider.ResourceInfo + for entry, err = dir.ReadDirPlus(goceph.StatxBasicStats, 0); entry != nil && err == nil; entry, err = dir.ReadDirPlus(goceph.StatxBasicStats, 0) { if fs.conf.HiddenDirs[entry.Name()] { continue @@ -271,8 +263,7 @@ func (fs *cephfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey ri, err = user.fileAsResourceInfo(cv, filepath.Join(path, entry.Name()), entry.Statx(), mdKeys) if ri == nil || err != nil { if err != nil { - log := appctx.GetLogger(ctx) - log.Err(err).Msg("cephfs: error in file as resource info") + log.Debug().Any("resourceInfo", ri).Err(err).Msg("fileAsResourceInfo returned") } err = nil continue @@ -282,7 +273,7 @@ func (fs *cephfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey } }) - return files, getRevaError(err) + return files, getRevaError(ctx, err) } func (fs *cephfs) Download(ctx context.Context, ref *provider.Reference) (rc io.ReadCloser, err error) { @@ -292,116 +283,31 @@ func (fs *cephfs) Download(ctx context.Context, ref *provider.Reference) (rc io. return nil, errors.Wrap(err, "cephfs: error resolving ref") } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { - if strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { - err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") + if rc, err = cv.mount.Open(path, os.O_RDONLY, 0); err != nil { + log.Debug().Any("ref", ref).Err(err).Msg("cv.mount.Open returned") return } - rc, err = cv.mount.Open(path, os.O_RDONLY, 0) }) - return rc, getRevaError(err) + return rc, getRevaError(ctx, err) } func (fs *cephfs) ListRevisions(ctx context.Context, ref *provider.Reference) (fvs []*provider.FileVersion, err error) { - //TODO(tmourati): Fix entry id logic - var path string - user := fs.makeUser(ctx) - if path, err = user.resolveRef(ref); err != nil { - return nil, errors.Wrap(err, "cephfs: error resolving ref") - } - - user.op(func(cv *cacheVal) { - if strings.HasPrefix(path, removeLeadingSlash(fs.conf.ShareFolder)) { - err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") - return - } - var dir *goceph.Directory - if dir, err = cv.mount.OpenDir(".snap"); err != nil { - return - } - defer closeDir(dir) - - for d, _ := dir.ReadDir(); d != nil; d, _ = dir.ReadDir() { - var revPath string - var stat Statx - var e error - - if strings.HasPrefix(d.Name(), ".") { - continue - } - - revPath, e = resolveRevRef(cv.mount, ref, d.Name()) - if e != nil { - continue - } - stat, e = cv.mount.Statx(revPath, goceph.StatxMtime|goceph.StatxSize, 0) - if e != nil { - continue - } - fvs = append(fvs, &provider.FileVersion{ - Key: d.Name(), - Size: stat.Size, - Mtime: uint64(stat.Mtime.Sec), - }) - } - }) - - return fvs, getRevaError(err) + return nil, errtypes.NotSupported("cephfs: RestoreRevision not supported") } func (fs *cephfs) DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (file io.ReadCloser, err error) { - //TODO(tmourati): Fix entry id logic - user := fs.makeUser(ctx) - - user.op(func(cv *cacheVal) { - var revPath string - revPath, err = resolveRevRef(cv.mount, ref, key) - if err != nil { - return - } - - file, err = cv.mount.Open(revPath, os.O_RDONLY, 0) - }) - - return file, getRevaError(err) + return nil, errtypes.NotSupported("cephfs: RestoreRevision not supported") } func (fs *cephfs) RestoreRevision(ctx context.Context, ref *provider.Reference, key string) (err error) { - //TODO(tmourati): Fix entry id logic - var path string - user := fs.makeUser(ctx) - if path, err = user.resolveRef(ref); err != nil { - return errors.Wrap(err, "cephfs: error resolving ref") - } - - user.op(func(cv *cacheVal) { - var revPath string - if revPath, err = resolveRevRef(cv.mount, ref, key); err != nil { - err = errors.Wrap(err, "cephfs: error resolving revision ref "+ref.String()) - return - } - - var src, dst *goceph.File - if src, err = cv.mount.Open(revPath, os.O_RDONLY, 0); err != nil { - return - } - defer closeFile(src) - - if dst, err = cv.mount.Open(path, os.O_WRONLY|os.O_TRUNC, 0); err != nil { - return - } - defer closeFile(dst) - - _, err = io.Copy(dst, src) - }) - - return getRevaError(err) + return errtypes.NotSupported("cephfs: RestoreRevision not supported") } func (fs *cephfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str string, err error) { - //TODO(tmourati): Add entry id logic - return "", errtypes.NotSupported("cephfs: entry IDs currently not supported") + return "", errtypes.NotSupported("cephfs: ids currently not supported") } func (fs *cephfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { @@ -411,11 +317,14 @@ func (fs *cephfs) AddGrant(ctx context.Context, ref *provider.Reference, g *prov return } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { - err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + if err = fs.changePerms(ctx, cv.mount, g, path, updateGrant); err != nil { + log.Debug().Any("ref", ref).Any("grant", g).Err(err).Msg("AddGrant returned") + } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { @@ -425,11 +334,14 @@ func (fs *cephfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *p return } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { - err = fs.changePerms(ctx, cv.mount, g, path, removeGrant) + if err = fs.changePerms(ctx, cv.mount, g, path, removeGrant); err != nil { + log.Debug().Any("ref", ref).Any("grant", g).Err(err).Msg("RemoveGrant returned") + } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { @@ -439,11 +351,14 @@ func (fs *cephfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *p return } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { - err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + if err = fs.changePerms(ctx, cv.mount, g, path, updateGrant); err != nil { + log.Debug().Any("ref", ref).Any("grant", g).Err(err).Msg("UpdateGrant returned") + } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) (err error) { @@ -453,12 +368,15 @@ func (fs *cephfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *pro return } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { grant := &provider.Grant{Grantee: g} //nil perms will remove the whole grant - err = fs.changePerms(ctx, cv.mount, grant, path, removeGrant) + if err = fs.changePerms(ctx, cv.mount, grant, path, removeGrant); err != nil { + log.Debug().Any("ref", ref).Any("grant", grant).Err(err).Msg("DenyGrant returned") + } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) ListGrants(ctx context.Context, ref *provider.Reference) (glist []*provider.Grant, err error) { @@ -476,7 +394,7 @@ func (fs *cephfs) ListGrants(ctx context.Context, ref *provider.Reference) (glis } }) - return glist, getRevaError(err) + return glist, getRevaError(ctx, err) } func (fs *cephfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, used uint64, err error) { @@ -499,28 +417,11 @@ func (fs *cephfs) GetQuota(ctx context.Context, ref *provider.Reference) (total } }) - return total, used, getRevaError(err) + return total, used, getRevaError(ctx, err) } func (fs *cephfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) (err error) { - user := fs.makeUser(ctx) - - user.op(func(cv *cacheVal) { - if !strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { - err = errors.New("cephfs: can't create reference outside a share folder") - } else { - err = cv.mount.MakeDir(path, fs.conf.DirPerms) - } - }) - if err != nil { - return getRevaError(err) - } - - user.op(func(cv *cacheVal) { - err = cv.mount.SetXattr(path, xattrRef, []byte(targetURI.String()), 0) - }) - - return getRevaError(err) + return errors.New("error: CreateReference not implemented") } func (fs *cephfs) Shutdown(ctx context.Context) (err error) { @@ -540,6 +441,7 @@ func (fs *cephfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Refere return err } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { for k, v := range md.Metadata { if !strings.HasPrefix(k, xattrUserNs) { @@ -547,12 +449,13 @@ func (fs *cephfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Refere } if e := cv.mount.SetXattr(path, k, []byte(v), 0); e != nil { err = errors.Wrap(err, e.Error()) + log.Debug().Any("ref", ref).Str("key", k).Any("v", v).Err(err).Msg("SetXattr returned") return } } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { @@ -562,6 +465,7 @@ func (fs *cephfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe return err } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { for _, key := range keys { if !strings.HasPrefix(key, xattrUserNs) { @@ -569,32 +473,33 @@ func (fs *cephfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe } if e := cv.mount.RemoveXattr(path, key); e != nil { err = errors.Wrap(err, e.Error()) + log.Debug().Any("ref", ref).Str("key", key).Err(err).Msg("RemoveXattr returned") return } } }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) TouchFile(ctx context.Context, ref *provider.Reference) error { user := fs.makeUser(ctx) path, err := user.resolveRef(ref) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } + log := appctx.GetLogger(ctx) user.op(func(cv *cacheVal) { var file *goceph.File defer closeFile(file) if file, err = cv.mount.Open(path, os.O_CREATE|os.O_WRONLY, fs.conf.FilePerms); err != nil { + log.Debug().Any("ref", ref).Err(err).Msg("Touch: Open returned") return } - - //TODO(tmourati): Add entry id logic }) - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) EmptyRecycle(ctx context.Context) error { @@ -656,7 +561,7 @@ func (fs *cephfs) SetLock(ctx context.Context, ref *provider.Reference, lock *pr user := fs.makeUser(ctx) path, err := user.resolveRef(ref) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } op := goceph.LockEX @@ -684,14 +589,14 @@ func (fs *cephfs) SetLock(ctx context.Context, ref *provider.Reference, lock *pr return fs.SetArbitraryMetadata(ctx, ref, md) } - return getRevaError(err) + return getRevaError(ctx, err) } func (fs *cephfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { user := fs.makeUser(ctx) path, err := user.resolveRef(ref) if err != nil { - return nil, getRevaError(err) + return nil, getRevaError(ctx, err) } var l *provider.Lock @@ -744,7 +649,7 @@ func (fs *cephfs) GetLock(ctx context.Context, ref *provider.Reference) (*provid return }) - return l, getRevaError(err) + return l, getRevaError(ctx, err) } // TODO(lopresti) part of this logic is duplicated from eosfs.go, should be factored out @@ -787,7 +692,7 @@ func (fs *cephfs) Unlock(ctx context.Context, ref *provider.Reference, lock *pro user := fs.makeUser(ctx) path, err := user.resolveRef(ref) if err != nil { - return getRevaError(err) + return getRevaError(ctx, err) } oldLock, err := fs.GetLock(ctx, ref) @@ -824,5 +729,5 @@ func (fs *cephfs) Unlock(ctx context.Context, ref *provider.Reference, lock *pro return fs.UnsetArbitraryMetadata(ctx, ref, []string{xattrLock}) } - return getRevaError(err) + return getRevaError(ctx, err) } diff --git a/pkg/storage/fs/cephfs/chunking.go b/pkg/storage/fs/cephfs/chunking.go index a89fa23824..1b5be2b4b0 100644 --- a/pkg/storage/fs/cephfs/chunking.go +++ b/pkg/storage/fs/cephfs/chunking.go @@ -26,6 +26,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "regexp" "strconv" @@ -43,11 +44,12 @@ func IsChunked(fn string) (bool, error) { } // ChunkBLOBInfo stores info about a particular chunk +// example: given /users/peter/myfile.txt-chunking-1234-10-2 type ChunkBLOBInfo struct { - Path string - TransferID string - TotalChunks int - CurrentChunk int + Path string // example: /users/peter/myfile.txt + TransferID string // example: 1234 + TotalChunks int // example: 10 + CurrentChunk int // example: 2 } // Not using the resource path in the chunk folder name allows uploading to @@ -85,21 +87,22 @@ func GetChunkBLOBInfo(path string) (*ChunkBLOBInfo, error) { // ChunkHandler manages chunked uploads, storing the chunks in a temporary directory // until it gets the final chunk which is then returned. type ChunkHandler struct { - user *User - chunkFolder string + user *User + uploadFolder string // example: /users/peter/.uploads } // NewChunkHandler creates a handler for chunked uploads. func NewChunkHandler(ctx context.Context, fs *cephfs) *ChunkHandler { - return &ChunkHandler{fs.makeUser(ctx), fs.conf.UploadFolder} + u := fs.makeUser(ctx) + return &ChunkHandler{u, path.Join(u.home, fs.conf.UploadFolder)} } -func (c *ChunkHandler) getChunkTempFileName() string { +func (c *ChunkHandler) getTempFileName() string { return fmt.Sprintf("__%d_%s", time.Now().Unix(), uuid.New().String()) } -func (c *ChunkHandler) getChunkFolderName(i *ChunkBLOBInfo) (path string, err error) { - path = filepath.Join(c.chunkFolder, i.uploadID()) +func (c *ChunkHandler) getAndCreateTransferFolderName(i *ChunkBLOBInfo) (path string, err error) { + path = filepath.Join(c.uploadFolder, i.uploadID()) c.user.op(func(cv *cacheVal) { err = cv.mount.MakeDir(path, 0777) }) @@ -107,6 +110,7 @@ func (c *ChunkHandler) getChunkFolderName(i *ChunkBLOBInfo) (path string, err er return } +// TODO(labkode): I don't like how this function looks like, better to refactor func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chunk string, err error) { chunkInfo, err := GetChunkBLOBInfo(path) if err != nil { @@ -114,10 +118,21 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chu return } - chunkTempFilename := c.getChunkTempFileName() + transferFolderName, err := c.getAndCreateTransferFolderName(chunkInfo) + if err != nil { + // TODO(labkode): skip error for now + // err = fmt.Errorf("error getting transfer folder anme", err) + return + } + + // here we write a temporary file that will be renamed to the transfer folder + // with the correct sequence number filename. + // we do not store this before-rename temporary files inside the transfer folder + // to avoid errors when counting the number of chunks for finalizing the transfer. + tmpFilename := c.getTempFileName() c.user.op(func(cv *cacheVal) { var tmpFile *goceph.File - target := filepath.Join(c.chunkFolder, chunkTempFilename) + target := filepath.Join(c.uploadFolder, tmpFilename) tmpFile, err = cv.mount.Open(target, os.O_CREATE|os.O_WRONLY, c.user.fs.conf.FilePerms) defer closeFile(tmpFile) if err != nil { @@ -129,15 +144,9 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chu return } - chunksFolderName, err := c.getChunkFolderName(chunkInfo) - if err != nil { - return - } - // c.logger.Info().Log("chunkfolder", chunksFolderName) - - chunkTarget := filepath.Join(chunksFolderName, strconv.Itoa(chunkInfo.CurrentChunk)) + chunkTarget := filepath.Join(transferFolderName, strconv.Itoa(chunkInfo.CurrentChunk)) c.user.op(func(cv *cacheVal) { - err = cv.mount.Rename(chunkTempFilename, chunkTarget) + err = cv.mount.Rename(tmpFilename, chunkTarget) }) if err != nil { return @@ -154,7 +163,7 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chu var entry *goceph.DirEntry var chunkFile, assembledFile *goceph.File - dir, err = cv.mount.OpenDir(chunksFolderName) + dir, err = cv.mount.OpenDir(transferFolderName) defer closeDir(dir) for entry, err = dir.ReadDir(); entry != nil && err == nil; entry, err = dir.ReadDir() { @@ -167,16 +176,20 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chu return } - chunk = filepath.Join(c.chunkFolder, c.getChunkTempFileName()) - assembledFile, err = cv.mount.Open(chunk, os.O_CREATE|os.O_WRONLY, c.user.fs.conf.FilePerms) + // from now on we do have all the necessary chunks, + // so we create a temporary file where all the chunks will be written + // before being renamed to the requested location, from the example: /users/peter/myfile.txt + + assemblyFilename := filepath.Join(c.uploadFolder, c.getTempFileName()) + assembledFile, err = cv.mount.Open(assemblyFilename, os.O_CREATE|os.O_WRONLY, c.user.fs.conf.FilePerms) defer closeFile(assembledFile) - defer deleteFile(cv.mount, chunk) + defer deleteFile(cv.mount, assemblyFilename) if err != nil { return } for i := 0; i < numEntries; i++ { - target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + target := filepath.Join(transferFolderName, strconv.Itoa(i)) chunkFile, err = cv.mount.Open(target, os.O_RDONLY, 0) if err != nil { @@ -189,22 +202,22 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chu } } - // necessary approach in case assembly fails + // clean all the chunks that made the assembly file for i := 0; i < numEntries; i++ { - target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + target := filepath.Join(transferFolderName, strconv.Itoa(i)) err = cv.mount.Unlink(target) if err != nil { return } } - _ = cv.mount.Unlink(chunksFolderName) }) - - return true, chunk, nil + return } // WriteChunk saves an intermediate chunk temporarily and assembles all chunks // once the final one is received. +// this function will return the original filename (myfile.txt) and the assemblyPath when +// the upload is completed func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { finish, chunk, err := c.saveChunk(fn, r) if err != nil { diff --git a/pkg/storage/fs/cephfs/connections.go b/pkg/storage/fs/cephfs/connections.go index 916c6b9a93..d18a288282 100644 --- a/pkg/storage/fs/cephfs/connections.go +++ b/pkg/storage/fs/cephfs/connections.go @@ -116,21 +116,21 @@ type adminConn struct { // radosIO *rados2.IOContext } -func newAdminConn(conf *Options) *adminConn { +func newAdminConn(conf *Options) (*adminConn, error) { rados, err := rados2.NewConnWithUser(conf.ClientID) if err != nil { - return nil + return nil, errors.Wrap(err, "error creating connection with user for client id: "+conf.ClientID) } if err = rados.ReadConfigFile(conf.Config); err != nil { - return nil + return nil, errors.Wrapf(err, "error reading config file %s", conf.Config) } if err = rados.SetConfigOption("keyring", conf.Keyring); err != nil { - return nil + return nil, errors.Wrapf(err, "error setting keyring conf: %s", conf.Keyring) } if err = rados.Connect(); err != nil { - return nil + return nil, errors.Wrap(err, "error connecting to rados") } // TODO: May use later for file ids @@ -166,13 +166,13 @@ func newAdminConn(conf *Options) *adminConn { mount, err := goceph.CreateFromRados(rados) if err != nil { rados.Shutdown() - return nil + return nil, errors.Wrap(err, "error calling CreateFromRados") } if err = mount.MountWithRoot(conf.Root); err != nil { rados.Shutdown() destroyCephConn(mount, nil) - return nil + return nil, errors.Wrapf(err, "error mounting with root %s", conf.Root) } return &adminConn{ @@ -181,7 +181,7 @@ func newAdminConn(conf *Options) *adminConn { mount, rados, // radosIO, - } + }, nil } func newConn(user *User) *cacheVal { @@ -203,6 +203,7 @@ func newConn(user *User) *cacheVal { } if user != nil { //nil creates admin conn + // TODO(lopresti) here we may need to impersonate a different user in order to support ACLs! perm = goceph.NewUserPerm(int(user.UidNumber), int(user.GidNumber), []int{}) if err = mount.SetMountPerms(perm); err != nil { return destroyCephConn(mount, perm) @@ -213,11 +214,14 @@ func newConn(user *User) *cacheVal { return destroyCephConn(mount, perm) } - if user != nil && !user.fs.conf.DisableHome { - if err = mount.ChangeDir(user.fs.conf.Root); err != nil { - return destroyCephConn(mount, perm) + // TODO(labkode): we leave the mount on the fs root + /* + if user != nil && !user.fs.conf.DisableHome { + if err = mount.ChangeDir(user.fs.conf.Root); err != nil { + return destroyCephConn(mount, perm) + } } - } + */ return &cacheVal{ perm: perm, diff --git a/pkg/storage/fs/cephfs/errors.go b/pkg/storage/fs/cephfs/errors.go index d866e6d837..8182635bea 100644 --- a/pkg/storage/fs/cephfs/errors.go +++ b/pkg/storage/fs/cephfs/errors.go @@ -29,7 +29,8 @@ package cephfs import "C" import ( "fmt" - + "context" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" ) @@ -45,10 +46,12 @@ var ( errPermissionDenied = wrapErrorMsg(C.EACCES) ) -func getRevaError(err error) error { +func getRevaError(ctx context.Context, err error) error { if err == nil { return nil } + log := appctx.GetLogger(ctx) + log.Warn().Err(err).Msg("cephfs error") switch err.Error() { case errNotFound: return errtypes.NotFound("cephfs: entry not found") diff --git a/pkg/storage/fs/cephfs/options.go b/pkg/storage/fs/cephfs/options.go index c33e76eb89..a7b6149462 100644 --- a/pkg/storage/fs/cephfs/options.go +++ b/pkg/storage/fs/cephfs/options.go @@ -22,25 +22,19 @@ package cephfs import ( - "path/filepath" - "github.com/cs3org/reva/pkg/sharedconf" ) // Options for the cephfs module type Options struct { - ClientID string `mapstructure:"client_id"` - Config string `mapstructure:"config"` - GatewaySvc string `mapstructure:"gatewaysvc"` - IndexPool string `mapstructure:"index_pool"` - Keyring string `mapstructure:"keyring"` - Root string `mapstructure:"root"` - ShadowFolder string `mapstructure:"shadow_folder"` - ShareFolder string `mapstructure:"share_folder"` - UploadFolder string `mapstructure:"uploads"` - UserLayout string `mapstructure:"user_layout"` - - DisableHome bool `mapstructure:"disable_home"` + ClientID string `mapstructure:"client_id"` + Config string `mapstructure:"config"` + GatewaySvc string `mapstructure:"gatewaysvc"` + IndexPool string `mapstructure:"index_pool"` + Keyring string `mapstructure:"keyring"` + Root string `mapstructure:"root"` + UploadFolder string `mapstructure:"uploads"` + UserLayout string `mapstructure:"user_layout"` DirPerms uint32 `mapstructure:"dir_perms"` FilePerms uint32 `mapstructure:"file_perms"` UserQuotaBytes uint64 `mapstructure:"user_quota_bytes"` @@ -71,27 +65,14 @@ func (c *Options) ApplyDefaults() { } if c.Root == "" { - c.Root = "/home" + c.Root = "/cephfs" } else { c.Root = addLeadingSlash(c.Root) } - if c.ShadowFolder == "" { - c.ShadowFolder = "/.reva_hidden" - } else { - c.ShadowFolder = addLeadingSlash(c.ShadowFolder) - } - - if c.ShareFolder == "" { - c.ShareFolder = "/Shares" - } else { - c.ShareFolder = addLeadingSlash(c.ShareFolder) - } - if c.UploadFolder == "" { c.UploadFolder = ".uploads" } - c.UploadFolder = filepath.Join(c.ShadowFolder, c.UploadFolder) if c.UserLayout == "" { c.UserLayout = "{{.Username}}" @@ -100,7 +81,7 @@ func (c *Options) ApplyDefaults() { c.HiddenDirs = map[string]bool{ ".": true, "..": true, - removeLeadingSlash(c.ShadowFolder): true, + removeLeadingSlash(c.UploadFolder): true, } if c.DirPerms == 0 { diff --git a/pkg/storage/fs/cephfs/upload.go b/pkg/storage/fs/cephfs/upload.go index 64ac7f7179..44639881a7 100644 --- a/pkg/storage/fs/cephfs/upload.go +++ b/pkg/storage/fs/cephfs/upload.go @@ -39,46 +39,52 @@ func (fs *cephfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read ok, err := IsChunked(p) if err != nil { - return errors.Wrap(err, "cephfs: error checking path") + return errors.Wrap(err, "cephfs: error checking if path is chunked") } - if ok { - var assembledFile string - p, assembledFile, err = NewChunkHandler(ctx, fs).WriteChunk(p, r) - if err != nil { - return err - } - if p == "" { - return errtypes.PartialContent(ref.String()) - } + + if !ok { + var file io.WriteCloser user.op(func(cv *cacheVal) { - r, err = cv.mount.Open(assembledFile, os.O_RDONLY, 0) - }) - if err != nil { - return errors.Wrap(err, "cephfs: error opening assembled file") - } - defer r.Close() - defer user.op(func(cv *cacheVal) { - _ = cv.mount.Unlink(assembledFile) + file, err = cv.mount.Open(p, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fs.conf.FilePerms) + if err != nil { + err = errors.Wrap(err, "cephfs: error opening binary file") + return + } + defer file.Close() + + _, err = io.Copy(file, r) + if err != nil { + err = errors.Wrap(err, "cephfs: error writing to binary file") + return + } }) + + return nil } - var file io.WriteCloser + // upload is chunked + + var assembledFile string + + // iniate the chunk handler + originalFilename, assembledFile, err := NewChunkHandler(ctx, fs).WriteChunk(p, r) + if err != nil { + return errors.Wrapf(err, "error writing chunk %v %v %v", p, r, assembledFile) + } + if originalFilename == "" { // means we wrote a chunk only + return errtypes.PartialContent(ref.String()) + } user.op(func(cv *cacheVal) { - file, err = cv.mount.Open(p, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fs.conf.FilePerms) - if err != nil { - err = errors.Wrap(err, "cephfs: error opening binary file") - return - } - defer file.Close() - - _, err = io.Copy(file, r) - if err != nil { - err = errors.Wrap(err, "cephfs: error writing to binary file") - return - } + err = cv.mount.Rename(assembledFile, originalFilename) + }) + if err != nil { + return errors.Wrap(err, "cephfs: error renaming assembled file") + } + defer user.op(func(cv *cacheVal) { + _ = cv.mount.Unlink(assembledFile) }) + return nil - return err } func (fs *cephfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { diff --git a/pkg/storage/fs/cephfs/user.go b/pkg/storage/fs/cephfs/user.go index da15b1ec45..a2d3cdeb97 100644 --- a/pkg/storage/fs/cephfs/user.go +++ b/pkg/storage/fs/cephfs/user.go @@ -53,11 +53,8 @@ type User struct { func (fs *cephfs) makeUser(ctx context.Context) *User { u := appctx.ContextMustGetUser(ctx) - home := fs.conf.Root - if !fs.conf.DisableHome { - home = filepath.Join(fs.conf.Root, templates.WithUser(u, fs.conf.UserLayout)) - } - + // home := fs.conf.Root + home := filepath.Join(fs.conf.Root, templates.WithUser(u, fs.conf.UserLayout)) return &User{u, fs, ctx, home} } @@ -139,8 +136,6 @@ func (user *User) fileAsResourceInfo(cv *cacheVal, path string, stat *goceph.Cep } } - //TODO(tmourati): Add entry id logic here - var etag string if isDir(_type) { rctime, _ := cv.mount.GetXattr(path, "ceph.dir.rctime") @@ -162,37 +157,11 @@ func (user *User) fileAsResourceInfo(cv *cacheVal, path string, stat *goceph.Cep } } + // cephfs does not provide checksums, so we cannot set it + // a 3rd party tool can add a checksum attribute and we can read it, + // if ever that is implemented. var checksum provider.ResourceChecksum - var md5 string - if _type == provider.ResourceType_RESOURCE_TYPE_FILE { - md5tsBA, err := cv.mount.GetXattr(path, xattrMd5ts) //local error inside if scope - if err == nil { - md5ts, _ := strconv.ParseInt(string(md5tsBA), 10, 64) - if stat.Mtime.Sec == md5ts { - md5BA, err := cv.mount.GetXattr(path, xattrMd5) - if err != nil { - md5, err = calcChecksum(path, cv.mount, stat) - } else { - md5 = string(md5BA) - } - } else { - md5, err = calcChecksum(path, cv.mount, stat) - } - } else { - md5, err = calcChecksum(path, cv.mount, stat) - } - - if err != nil && err.Error() == errPermissionDenied { - checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET - } else if err != nil { - return nil, errors.New("cephfs: error calculating checksum of file") - } else { - checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5 - checksum.Sum = md5 - } - } else { - checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET - } + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET var ownerID *userv1beta1.UserId if stat.Uid != 0 { @@ -230,13 +199,13 @@ func (user *User) fileAsResourceInfo(cv *cacheVal, path string, stat *goceph.Cep return } -func (user *User) resolveRef(ref *provider.Reference) (str string, err error) { +func (user *User) resolveRef(ref *provider.Reference) (string, error) { if ref == nil { - return "", fmt.Errorf("cephfs: nil reference") + return "", fmt.Errorf("cephfs: nil reference provided") } - if str = ref.GetPath(); str == "" { - return "", errtypes.NotSupported("cephfs: entry IDs not currently supported") + if ref.GetPath() == "" { + return "", errtypes.NotSupported("cephfs: path not provided, id based refs are not supported") } - return + return ref.GetPath(), nil } diff --git a/pkg/storage/fs/cephfs/utils.go b/pkg/storage/fs/cephfs/utils.go index 8649e5323b..fa868ec895 100644 --- a/pkg/storage/fs/cephfs/utils.go +++ b/pkg/storage/fs/cephfs/utils.go @@ -22,13 +22,7 @@ package cephfs import ( - "crypto/md5" - "encoding/hex" - "fmt" - "io" - "os" "path/filepath" - "strconv" goceph "github.com/ceph/go-ceph/cephfs" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -74,75 +68,6 @@ func isDir(t provider.ResourceType) bool { return t == provider.ResourceType_RESOURCE_TYPE_CONTAINER } -// TODO: Use when fileids are available -/* -func (fs *cephfs) makeFIDPath(fid string) string { - return "" // filepath.Join(fs.conf.EIDFolder, fid) EIDFolder does not exist -} - -func (fs *cephfs) makeFID(absolutePath string, inode string) (rid *provider.ResourceId, err error) { - sum := md5.New() - sum.Write([]byte(absolutePath)) - fid := fmt.Sprintf("%s-%s", hex.EncodeToString(sum.Sum(nil)), inode) - rid = &provider.ResourceId{OpaqueId: fid} - - _ = fs.adminConn.adminMount.Link(absolutePath, fs.makeFIDPath(fid)) - _ = fs.adminConn.adminMount.SetXattr(absolutePath, xattrEID, []byte(fid), 0) - - return -} - -func (fs *cephfs) getFIDPath(cv *cacheVal, path string) (fid string, err error) { - var buffer []byte - if buffer, err = cv.mount.GetXattr(path, xattrEID); err != nil { - return - } - - return fs.makeFIDPath(string(buffer)), err -} -*/ - -func calcChecksum(filepath string, mt Mount, stat Statx) (checksum string, err error) { - file, err := mt.Open(filepath, os.O_RDONLY, 0) - defer closeFile(file) - if err != nil { - return - } - hash := md5.New() - if _, err = io.Copy(hash, file); err != nil { - return - } - checksum = hex.EncodeToString(hash.Sum(nil)) - // we don't care if they fail, the checksum will just be recalculated if an error happens - _ = mt.SetXattr(filepath, xattrMd5ts, []byte(strconv.FormatInt(stat.Mtime.Sec, 10)), 0) - _ = mt.SetXattr(filepath, xattrMd5, []byte(checksum), 0) - - return -} - -func resolveRevRef(mt Mount, ref *provider.Reference, revKey string) (str string, err error) { - var buf []byte - if ref.GetResourceId() != nil { - str, err = mt.Readlink(filepath.Join(snap, revKey, ref.ResourceId.OpaqueId)) - if err != nil { - return "", fmt.Errorf("cephfs: invalid reference %+v", ref) - } - } else if str = ref.GetPath(); str != "" { - buf, err = mt.GetXattr(str, xattrEID) - if err != nil { - return - } - str, err = mt.Readlink(filepath.Join(snap, revKey, string(buf))) - if err != nil { - return - } - } else { - return "", fmt.Errorf("cephfs: empty reference %+v", ref) - } - - return filepath.Join(snap, revKey, str), err -} - func removeLeadingSlash(path string) string { return filepath.Join(".", path) } @@ -182,6 +107,7 @@ func pathGenerator(path string, reverse bool, str chan string) { func walkPath(path string, f func(string) error, reverse bool) (err error) { paths := make(chan string) + // TODO(labkode): carefully review this, a race could happen if pathGenerator gorouting is slow go pathGenerator(path, reverse, paths) for path := range paths { if path == "" { @@ -196,55 +122,3 @@ func walkPath(path string, f func(string) error, reverse bool) (err error) { return } - -// TODO: Use when fileids are available -/* -func (fs *cephfs) writeIndex(oid string, value string) (err error) { - return fs.adminConn.radosIO.WriteFull(oid, []byte(value)) -} - -func (fs *cephfs) removeIndex(oid string) error { - return fs.adminConn.radosIO.Delete(oid) -} - -func (fs *cephfs) resolveIndex(oid string) (fullPath string, err error) { - var i int - var currPath strings.Builder - root := string(filepath.Separator) - offset := uint64(0) - io := fs.adminConn.radosIO - bsize := 4096 - buffer := make([]byte, bsize) - for { - for { //read object - i, err = io.Read(oid, buffer, offset) - offset += uint64(bsize) - currPath.Write(buffer) - if err == nil && i >= bsize { - buffer = buffer[:0] - continue - } else { - offset = 0 - break - } - } - if err != nil { - return - } - - ss := strings.SplitN(currPath.String(), string(filepath.Separator), 2) - if len(ss) != 2 { - if currPath.String() == root { - return - } - - return "", fmt.Errorf("cephfs: entry id is not in the form of \"parentID/entryname\"") - } - parentOID := ss[0] - entryName := ss[1] - fullPath = filepath.Join(entryName, fullPath) - oid = parentOID - currPath.Reset() - } -} -*/ diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index bb6dbbae96..6b818672e8 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -66,6 +66,7 @@ const ( lwShareAttrKey = "reva.lwshare" // used to store grants to lightweight accounts lockPayloadKey = "reva.lockpayload" // used to store lock payloads eosLockKey = "app.lock" // this is the key known by EOS to enforce a lock. + FavoritesKey = "http://owncloud.org/ns/favorite" ) const ( @@ -193,6 +194,7 @@ func NewEOSFS(ctx context.Context, c *Config) (storage.FS, error) { VersionInvariant: c.VersionInvariant, ReadUsesLocalTemp: c.ReadUsesLocalTemp, WriteUsesLocalTemp: c.WriteUsesLocalTemp, + TokenExpiry: c.TokenExpiry, } eosHTTPOpts := &eosgrpc.HTTPOptions{ BaseURL: c.MasterURL, @@ -2272,6 +2274,8 @@ func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) ( } } + parseAndSetFavoriteAttr(ctx, filteredAttrs) + info := &provider.ResourceInfo{ Id: &provider.ResourceId{OpaqueId: fmt.Sprintf("%d", eosFileInfo.Inode)}, Path: p, @@ -2508,6 +2512,29 @@ func (fs *eosfs) getEosMetadata(finfo *eosclient.FileInfo) []byte { return v } +func parseAndSetFavoriteAttr(ctx context.Context, attrs map[string]string) { + // Read and correctly set the favorite attr + if user, ok := appctx.ContextGetUser(ctx); ok { + if favAttrStr, ok := attrs[FavoritesKey]; ok { + favUsers, err := acl.Parse(favAttrStr, acl.ShortTextForm) + if err != nil { + return + } + for _, u := range favUsers.Entries { + // Check if the current user has favorited this resource + if u.Qualifier == user.Id.OpaqueId { + // Set attr val to 1 + attrs[FavoritesKey] = "1" + return + } + } + } + } + + // Delete the favorite attr from the response + delete(attrs, FavoritesKey) +} + /* Merge shadow on requests for /home ? diff --git a/pkg/user/manager/ldap/ldap.go b/pkg/user/manager/ldap/ldap.go index 31dbf8d2af..0973f0a99e 100644 --- a/pkg/user/manager/ldap/ldap.go +++ b/pkg/user/manager/ldap/ldap.go @@ -374,6 +374,9 @@ func (m *manager) FindUsers(ctx context.Context, query string, skipFetchingGroup } func (m *manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]string, error) { + if m.c.GroupFilter == "" { + return []string{}, nil + } l, err := utils.GetLDAPConnection(&m.c.LDAPConn) if err != nil { return []string{}, err