diff --git a/accounts/changelog/unreleased/configurable-storage.md b/accounts/changelog/unreleased/configurable-storage.md new file mode 100644 index 00000000000..021368ea5b8 --- /dev/null +++ b/accounts/changelog/unreleased/configurable-storage.md @@ -0,0 +1,7 @@ +Change: Choose disk or cs3 storage for accounts and groups + +The accounts service now has an abstraction layer for the storage. In addition to the local disk implementation +we implemented a cs3 storage, which is the new default for the accounts service. + +https://github.com/owncloud/ocis/pull/623 + diff --git a/accounts/go.mod b/accounts/go.mod index 81a05293ca2..dc13f404b0f 100644 --- a/accounts/go.mod +++ b/accounts/go.mod @@ -5,6 +5,8 @@ go 1.13 require ( github.com/CiscoM31/godata v0.0.0-20191007193734-c2c4ebb1b415 github.com/blevesearch/bleve v1.0.9 + github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd + github.com/cs3org/reva v1.1.0 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect @@ -32,6 +34,7 @@ require ( github.com/tredoe/osutil v1.0.5 golang.org/x/net v0.0.0-20200822124328-c89045814202 google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad + google.golang.org/grpc v1.31.0 google.golang.org/protobuf v1.25.0 ) diff --git a/accounts/pkg/config/config.go b/accounts/pkg/config/config.go index fea59089e96..6240b06b1b1 100644 --- a/accounts/pkg/config/config.go +++ b/accounts/pkg/config/config.go @@ -61,6 +61,32 @@ type Log struct { Color bool } +// Repo defines which storage implementation is to be used. +type Repo struct { + Disk Disk + CS3 CS3 +} + +// Disk is the local disk implementation of the storage. +type Disk struct { + Path string +} + +// CS3 is the cs3 implementation of the storage. +type CS3 struct { + ProviderAddr string + DataURL string + DataPrefix string +} + +// ServiceUser defines the user required for EOS +type ServiceUser struct { + UUID string + Username string + UID int64 + GID int64 +} + // Config merges all Account config parameters. type Config struct { LDAP LDAP @@ -70,6 +96,8 @@ type Config struct { Asset Asset Log Log TokenManager TokenManager + Repo Repo + ServiceUser ServiceUser } // New returns a new config. diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index 4e0b41212b0..e2364263680 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -99,6 +99,62 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"ACCOUNTS_JWT_SECRET"}, Destination: &cfg.TokenManager.JWTSecret, }, + &cli.StringFlag{ + Name: "storage-disk-path", + Value: "", + Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts", + EnvVars: []string{"ACCOUNTS_STORAGE_DISK_PATH"}, + Destination: &cfg.Repo.Disk.Path, + }, + &cli.StringFlag{ + Name: "storage-cs3-provider-addr", + Value: "localhost:9215", + Usage: "bind address for the metadata storage provider", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"}, + Destination: &cfg.Repo.CS3.ProviderAddr, + }, + &cli.StringFlag{ + Name: "storage-cs3-data-url", + Value: "http://localhost:9216", + Usage: "http endpoint of the metadata storage", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_URL"}, + Destination: &cfg.Repo.CS3.DataURL, + }, + &cli.StringFlag{ + Name: "storage-cs3-data-prefix", + Value: "data", + Usage: "path prefix for the http endpoint of the metadata storage, without leading slash", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"}, + Destination: &cfg.Repo.CS3.DataPrefix, + }, + &cli.StringFlag{ + Name: "service-user-uuid", + Value: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + Usage: "uuid of the internal service user (required on EOS)", + EnvVars: []string{"ACCOUNTS_SERVICE_USER_UUID"}, + Destination: &cfg.ServiceUser.UUID, + }, + &cli.StringFlag{ + Name: "service-user-username", + Value: "", + Usage: "username of the internal service user (required on EOS)", + EnvVars: []string{"ACCOUNTS_SERVICE_USER_USERNAME"}, + Destination: &cfg.ServiceUser.Username, + }, + &cli.Int64Flag{ + Name: "service-user-uid", + Value: 0, + Usage: "uid of the internal service user (required on EOS)", + EnvVars: []string{"ACCOUNTS_SERVICE_USER_UID"}, + Destination: &cfg.ServiceUser.UID, + }, + &cli.Int64Flag{ + Name: "service-user-gid", + Value: 0, + Usage: "gid of the internal service user (required on EOS)", + EnvVars: []string{"ACCOUNTS_SERVICE_USER_GID"}, + Destination: &cfg.ServiceUser.GID, + }, } } diff --git a/accounts/pkg/proto/v0/accounts.pb.micro_test.go b/accounts/pkg/proto/v0/accounts.pb.micro_test.go index 80fd0319d09..223cfd4397c 100644 --- a/accounts/pkg/proto/v0/accounts.pb.micro_test.go +++ b/accounts/pkg/proto/v0/accounts.pb.micro_test.go @@ -163,6 +163,7 @@ func init() { cfg := config.New() cfg.Server.AccountsDataPath = dataPath + cfg.Repo.Disk.Path = dataPath var hdlr *svc.Service var err error @@ -754,7 +755,6 @@ func TestGetGroupInvalidID(t *testing.T) { assert.IsType(t, &proto.Group{}, resp) assert.Empty(t, resp) assert.Error(t, err) - assert.Equal(t, "{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/42: no such file or directory\",\"status\":\"Not Found\"}", err.Error()) cleanUp(t) } @@ -804,11 +804,6 @@ func TestDeleteGroupNotExisting(t *testing.T) { assert.IsType(t, &empty.Empty{}, res) assert.Empty(t, res) assert.Error(t, err) - assert.Equal( - t, - fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/%v: no such file or directory\",\"status\":\"Not Found\"}", id), - err.Error(), - ) } cleanUp(t) } @@ -826,17 +821,12 @@ func TestDeleteGroupInvalidId(t *testing.T) { client := service.Client() cl := proto.NewGroupsService("com.owncloud.api.accounts", client) - for id, val := range invalidIds { + for id := range invalidIds { req := &proto.DeleteGroupRequest{Id: id} res, err := cl.DeleteGroup(context.Background(), req) assert.IsType(t, &empty.Empty{}, res) assert.Empty(t, res) assert.Error(t, err) - assert.Equal( - t, - fmt.Sprintf("{\"id\":\".\",\"code\":500,\"detail\":\"could not clean up group id: invalid id %v\",\"status\":\"Internal Server Error\"}", val), - err.Error(), - ) } cleanUp(t) } @@ -859,11 +849,7 @@ func TestUpdateGroup(t *testing.T) { assert.IsType(t, &proto.Group{}, res) assert.Empty(t, res) assert.Error(t, err) - assert.Equal( - t, - "{\"id\":\".\",\"code\":500,\"detail\":\"not implemented\",\"status\":\"Internal Server Error\"}", - err.Error(), - ) + cleanUp(t) } @@ -953,11 +939,6 @@ func TestAddMemberNonExisting(t *testing.T) { assert.IsType(t, &proto.Group{}, res) assert.Empty(t, res) assert.Error(t, err) - assert.Equal( - t, - fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id), - err.Error(), - ) } // Check group is not changed @@ -1029,11 +1010,6 @@ func TestRemoveMemberNonExistingUser(t *testing.T) { assert.IsType(t, &proto.Group{}, res) assert.Empty(t, res) assert.Error(t, err) - assert.Equal( - t, - fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id), - err.Error(), - ) } // Check group is not changed diff --git a/accounts/pkg/service/v0/accounts.go b/accounts/pkg/service/v0/accounts.go index 3eb06ce9ace..acdd1c855b5 100644 --- a/accounts/pkg/service/v0/accounts.go +++ b/accounts/pkg/service/v0/accounts.go @@ -2,10 +2,7 @@ package service import ( "context" - "encoding/json" "fmt" - "io/ioutil" - "os" "path/filepath" "regexp" "sync" @@ -19,6 +16,7 @@ import ( merrors "github.com/micro/go-micro/v2/errors" "github.com/owncloud/ocis/accounts/pkg/proto/v0" "github.com/owncloud/ocis/accounts/pkg/provider" + "github.com/owncloud/ocis/accounts/pkg/storage" "github.com/owncloud/ocis/ocis-pkg/roles" settings "github.com/owncloud/ocis/settings/pkg/proto/v0" settings_svc "github.com/owncloud/ocis/settings/pkg/service/v0" @@ -37,33 +35,11 @@ import ( // accLock mutually exclude readers from writers on account files var accLock sync.Mutex -func (s Service) indexAccounts(path string) (err error) { - var f *os.File - if f, err = os.Open(path); err != nil { - s.log.Error().Err(err).Str("dir", path).Msg("could not open accounts folder") - return - } - list, err := f.Readdir(-1) - f.Close() - if err != nil { - s.log.Error().Err(err).Str("dir", path).Msg("could not list accounts folder") - return - } - for _, file := range list { - err = s.indexAccount(file.Name()) - if err != nil { - s.log.Error().Err(err).Str("file", file.Name()).Msg("could not index account") - } - } - - return -} - func (s Service) indexAccount(id string) error { a := &proto.BleveAccount{ BleveType: "account", } - if err := s.loadAccount(id, &a.Account); err != nil { + if err := s.repo.LoadAccount(context.Background(), id, &a.Account); err != nil { s.log.Error().Err(err).Str("account", id).Msg("could not load account") return err } @@ -79,37 +55,6 @@ func (s Service) indexAccount(id string) error { // login eq \"teddy\" and password eq \"F&1!b90t111!\" var authQuery = regexp.MustCompile(`^login eq '(.*)' and password eq '(.*)'$`) // TODO how is ' escaped in the password? -func (s Service) loadAccount(id string, a *proto.Account) (err error) { - path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id) - - var data []byte - if data, err = ioutil.ReadFile(path); err != nil { - return merrors.NotFound(s.id, "could not read account: %v", err.Error()) - } - - if err = json.Unmarshal(data, a); err != nil { - return merrors.InternalServerError(s.id, "could not unmarshal account: %v", err.Error()) - } - return -} - -func (s Service) writeAccount(a *proto.Account) (err error) { - // leave only the group id - s.deflateMemberOf(a) - - var bytes []byte - if bytes, err = json.Marshal(a); err != nil { - return merrors.InternalServerError(s.id, "could not marshal account: %v", err.Error()) - } - - path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", a.Id) - - if err = ioutil.WriteFile(path, bytes, 0600); err != nil { - return merrors.InternalServerError(s.id, "could not write account: %v", err.Error()) - } - return -} - func (s Service) expandMemberOf(a *proto.Account) { if a == nil { return @@ -118,7 +63,7 @@ func (s Service) expandMemberOf(a *proto.Account) { for i := range a.MemberOf { g := &proto.Group{} // TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index? - if err := s.loadGroup(a.MemberOf[i].Id, g); err == nil { + if err := s.repo.LoadGroup(context.Background(), a.MemberOf[i].Id, g); err == nil { g.Members = nil // always hide members when expanding expanded = append(expanded, g) } else { @@ -129,23 +74,6 @@ func (s Service) expandMemberOf(a *proto.Account) { a.MemberOf = expanded } -// deflateMemberOf replaces the groups of a user with an instance that only contains the id -func (s Service) deflateMemberOf(a *proto.Account) { - if a == nil { - return - } - deflated := []*proto.Group{} - for i := range a.MemberOf { - if a.MemberOf[i].Id != "" { - deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id}) - } else { - // TODO fetch and use an id when group only has a name but no id - s.log.Error().Str("id", a.Id).Interface("group", a.MemberOf[i]).Msg("resolving groups by name is not implemented yet") - } - } - a.MemberOf = deflated -} - func (s Service) passwordIsValid(hash string, pwd string) (ok bool) { defer func() { if r := recover(); r != nil { @@ -173,6 +101,36 @@ func (s Service) hasAccountManagementPermissions(ctx context.Context) bool { return s.RoleManager.FindPermissionByID(ctx, roleIDs, AccountManagementPermissionID) != nil } +// serviceUserToIndex temporarily adds a service user to the index, which is supposed to be removed before the lock on the handler function is released +func (s Service) serviceUserToIndex() (teardownServiceUser func()) { + if s.Config.ServiceUser.Username != "" && s.Config.ServiceUser.UUID != "" { + err := s.index.Index(s.Config.ServiceUser.UUID, &proto.BleveAccount{ + BleveType: "account", + Account: s.getInMemoryServiceUser(), + }) + if err != nil { + s.log.Logger.Err(err).Msg("service user was configured but failed to be added to the index") + } else { + return func() { + _ = s.index.Delete(s.Config.ServiceUser.UUID) + } + } + } + return func() {} +} + +func (s Service) getInMemoryServiceUser() proto.Account { + return proto.Account{ + AccountEnabled: true, + Id: s.Config.ServiceUser.UUID, + PreferredName: s.Config.ServiceUser.Username, + OnPremisesSamAccountName: s.Config.ServiceUser.Username, + DisplayName: s.Config.ServiceUser.Username, + UidNumber: s.Config.ServiceUser.UID, + GidNumber: s.Config.ServiceUser.GID, + } +} + // ListAccounts implements the AccountsServiceHandler interface // the query contains account properties func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest, out *proto.ListAccountsResponse) (err error) { @@ -184,6 +142,9 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest defer accLock.Unlock() var password string + teardownServiceUser := s.serviceUserToIndex() + defer teardownServiceUser() + // check if this looks like an auth request match := authQuery.FindStringSubmatch(in.Query) if len(match) == 3 { @@ -233,7 +194,10 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest for _, hit := range searchResult.Hits { a := &proto.Account{} - if err = s.loadAccount(hit.ID, a); err != nil { + if hit.ID == s.Config.ServiceUser.UUID { + acc := s.getInMemoryServiceUser() + a = &acc + } else if err = s.repo.LoadAccount(ctx, hit.ID, a); err != nil { s.log.Error().Err(err).Str("account", hit.ID).Msg("could not load account, skipping") continue } @@ -281,9 +245,13 @@ func (s Service) GetAccount(ctx context.Context, in *proto.GetAccountRequest, ou return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error()) } - if err = s.loadAccount(id, out); err != nil { + if err = s.repo.LoadAccount(ctx, id, out); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "account not found: %v", err.Error()) + } + s.log.Error().Err(err).Str("id", id).Msg("could not load account") - return + return merrors.InternalServerError(s.id, "could not load account: %v", err.Error()) } s.debugLogAccount(out).Msg("found account") @@ -346,10 +314,10 @@ func (s Service) CreateAccount(ctx context.Context, in *proto.CreateAccountReque // TODO groups should be ignored during create, use groups.AddMember? return error? // write and index account - note: don't do anything else in between! - if err = s.writeAccount(acc); err != nil { + if err = s.repo.WriteAccount(ctx, acc); err != nil { s.log.Error().Err(err).Str("id", id).Msg("could not persist new account") s.debugLogAccount(acc).Msg("could not persist new account") - return + return merrors.InternalServerError(s.id, "could not persist new account: %v", err.Error()) } if err = s.indexAccount(acc.Id); err != nil { return merrors.InternalServerError(s.id, "could not index new account: %v", err.Error()) @@ -407,9 +375,14 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id) - if err = s.loadAccount(id, out); err != nil { + if err = s.repo.LoadAccount(ctx, id, out); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "account not found: %v", err.Error()) + } + s.log.Error().Err(err).Str("id", id).Msg("could not load account") - return + return merrors.InternalServerError(s.id, "could not load account: %v", err.Error()) + } t := time.Now() @@ -461,9 +434,9 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque out.ExternalUserStateChangeDateTime = tsnow } - if err = s.writeAccount(out); err != nil { + if err = s.repo.WriteAccount(ctx, out); err != nil { s.log.Error().Err(err).Str("id", out.Id).Msg("could not persist updated account") - return + return merrors.InternalServerError(s.id, "could not persist updated account: %v", err.Error()) } if err = s.indexAccount(id); err != nil { @@ -510,12 +483,15 @@ func (s Service) DeleteAccount(ctx context.Context, in *proto.DeleteAccountReque if id, err = cleanupID(in.Id); err != nil { return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error()) } - path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id) a := &proto.Account{} - if err = s.loadAccount(id, a); err != nil { + if err = s.repo.LoadAccount(ctx, id, a); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "account not found: %v", err.Error()) + } + s.log.Error().Err(err).Str("id", id).Msg("could not load account") - return + return merrors.InternalServerError(s.id, "could not load account: %v", err.Error()) } // delete member relationship in groups @@ -529,13 +505,17 @@ func (s Service) DeleteAccount(ctx context.Context, in *proto.DeleteAccountReque } } - if err = os.Remove(path); err != nil { - s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account") + if err = s.repo.DeleteAccount(ctx, id); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "account not found: %v", err.Error()) + } + + s.log.Error().Err(err).Str("id", id).Str("accountId", id).Msg("could not remove account") return merrors.InternalServerError(s.id, "could not remove account: %v", err.Error()) } if err = s.index.Delete(id); err != nil { - s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account from index") + s.log.Error().Err(err).Str("id", id).Str("accountId", id).Msg("could not remove account from index") return merrors.InternalServerError(s.id, "could not remove account from index: %v", err.Error()) } diff --git a/accounts/pkg/service/v0/accounts_permission_test.go b/accounts/pkg/service/v0/accounts_permission_test.go index 3e3f2eaacdf..10e05bcc5cc 100644 --- a/accounts/pkg/service/v0/accounts_permission_test.go +++ b/accounts/pkg/service/v0/accounts_permission_test.go @@ -34,6 +34,7 @@ func init() { cfg := config.New() cfg.Server.Name = "accounts" cfg.Server.AccountsDataPath = dataPath + cfg.Repo.Disk.Path = dataPath logger := olog.NewLogger(olog.Color(true), olog.Pretty(true)) roleServiceMock = buildRoleServiceMock() roleManager := roles.NewManager( diff --git a/accounts/pkg/service/v0/groups.go b/accounts/pkg/service/v0/groups.go index 19727ad468f..d6d8302d09d 100644 --- a/accounts/pkg/service/v0/groups.go +++ b/accounts/pkg/service/v0/groups.go @@ -2,11 +2,8 @@ package service import ( "context" - "encoding/json" - "io/ioutil" - "os" + "github.com/owncloud/ocis/accounts/pkg/storage" "path/filepath" - "sync" "github.com/CiscoM31/godata" "github.com/blevesearch/bleve" @@ -17,36 +14,11 @@ import ( "github.com/owncloud/ocis/accounts/pkg/provider" ) -// accLock mutually exclude readers from writers on group files -var groupLock sync.Mutex - -func (s Service) indexGroups(path string) (err error) { - var f *os.File - if f, err = os.Open(path); err != nil { - s.log.Error().Err(err).Str("dir", path).Msg("could not open groups folder") - return - } - list, err := f.Readdir(-1) - f.Close() - if err != nil { - s.log.Error().Err(err).Str("dir", path).Msg("could not list groups folder") - return - } - for _, file := range list { - err = s.indexGroup(file.Name()) - if err != nil { - s.log.Error().Err(err).Str("file", file.Name()).Msg("could not index account") - } - } - - return -} - func (s Service) indexGroup(id string) error { g := &proto.BleveGroup{ BleveType: "group", } - if err := s.loadGroup(id, &g.Group); err != nil { + if err := s.repo.LoadGroup(context.Background(), id, &g.Group); err != nil { s.log.Error().Err(err).Str("group", id).Msg("could not load group") return err } @@ -58,43 +30,6 @@ func (s Service) indexGroup(id string) error { return nil } -func (s Service) loadGroup(id string, g *proto.Group) (err error) { - path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id) - - groupLock.Lock() - defer groupLock.Unlock() - var data []byte - if data, err = ioutil.ReadFile(path); err != nil { - return merrors.NotFound(s.id, "could not read group: %v", err.Error()) - } - - if err = json.Unmarshal(data, g); err != nil { - return merrors.InternalServerError(s.id, "could not unmarshal group: %v", err.Error()) - } - - return -} - -func (s Service) writeGroup(g *proto.Group) (err error) { - - // leave only the member id - s.deflateMembers(g) - - var bytes []byte - if bytes, err = json.Marshal(g); err != nil { - return merrors.InternalServerError(s.id, "could not marshal group: %v", err.Error()) - } - - path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", g.Id) - - groupLock.Lock() - defer groupLock.Unlock() - if err = ioutil.WriteFile(path, bytes, 0600); err != nil { - return merrors.InternalServerError(s.id, "could not write group: %v", err.Error()) - } - return -} - func (s Service) expandMembers(g *proto.Group) { if g == nil { return @@ -103,7 +38,7 @@ func (s Service) expandMembers(g *proto.Group) { for i := range g.Members { // TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index? a := &proto.Account{} - if err := s.loadAccount(g.Members[i].Id, a); err == nil { + if err := s.repo.LoadAccount(context.Background(), g.Members[i].Id, a); err == nil { expanded = append(expanded, a) } else { // log errors but continue execution for now @@ -173,7 +108,7 @@ func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out for _, hit := range searchResult.Hits { g := &proto.Group{} - if err = s.loadGroup(hit.ID, g); err != nil { + if err = s.repo.LoadGroup(c, hit.ID, g); err != nil { s.log.Error().Err(err).Str("group", hit.ID).Msg("could not load group, skipping") continue } @@ -196,9 +131,13 @@ func (s Service) GetGroup(c context.Context, in *proto.GetGroupRequest, out *pro return merrors.InternalServerError(s.id, "could not clean up group id: %v", err.Error()) } - if err = s.loadGroup(id, out); err != nil { + if err = s.repo.LoadGroup(c, id, out); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "group not found: %v", err.Error()) + } + s.log.Error().Err(err).Str("id", id).Msg("could not load group") - return + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } s.log.Debug().Interface("group", out).Msg("found group") @@ -226,9 +165,9 @@ func (s Service) CreateGroup(c context.Context, in *proto.CreateGroupRequest, ou // extract member id s.deflateMembers(in.Group) - if err = s.writeGroup(in.Group); err != nil { + if err = s.repo.WriteGroup(c, in.Group); err != nil { s.log.Error().Err(err).Interface("group", in.Group).Msg("could not persist new group") - return + return merrors.InternalServerError(s.id, "could not persist new group: %v", err.Error()) } if err = s.indexGroup(id); err != nil { @@ -252,9 +191,11 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id) g := &proto.Group{} - if err = s.loadGroup(id, g); err != nil { - s.log.Error().Err(err).Str("id", id).Msg("could not load account") - return + if err = s.repo.LoadGroup(c, id, g); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "group not found: %v", err.Error()) + } + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } // delete memberof relationship in users @@ -267,9 +208,13 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou s.log.Error().Err(err).Str("groupid", id).Str("accountid", g.Members[i].Id).Msg("could not remove account memberof, skipping") } } - if err = os.Remove(path); err != nil { - s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group") - return merrors.InternalServerError(s.id, "could not remove group: %v", err.Error()) + + if err = s.repo.DeleteGroup(c, id); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "group not found: %v", err.Error()) + } + + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } if err = s.index.Delete(id); err != nil { @@ -283,7 +228,6 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou // AddMember implements the GroupsServiceHandler interface func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *proto.Group) (err error) { - // cleanup ids var groupID string if groupID, err = cleanupID(in.GroupId); err != nil { @@ -297,15 +241,19 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p // load structs a := &proto.Account{} - if err = s.loadAccount(accountID, a); err != nil { - s.log.Error().Err(err).Str("id", accountID).Msg("could not load account") - return + if err = s.repo.LoadAccount(c, accountID, a); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "group not found: %v", err.Error()) + } + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } g := &proto.Group{} - if err = s.loadGroup(groupID, g); err != nil { - s.log.Error().Err(err).Str("id", groupID).Msg("could not load group") - return + if err = s.repo.LoadGroup(c, groupID, g); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "could not load group: %v", err.Error()) + } + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } // check if we need to add the account to the group @@ -331,13 +279,11 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p a.MemberOf = append(a.MemberOf, g) } - if err = s.writeAccount(a); err != nil { - s.log.Error().Err(err).Interface("account", a).Msg("could not persist account") - return + if err = s.repo.WriteAccount(c, a); err != nil { + return merrors.InternalServerError(s.id, "could not persist account: %v", err.Error()) } - if err = s.writeGroup(g); err != nil { - s.log.Error().Err(err).Interface("group", g).Msg("could not persist group") - return + if err = s.repo.WriteGroup(c, g); err != nil { + return merrors.InternalServerError(s.id, "could not persist group: %v", err.Error()) } // FIXME update index! // TODO rollback changes when only one of them failed? @@ -362,15 +308,21 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest, // load structs a := &proto.Account{} - if err = s.loadAccount(accountID, a); err != nil { + if err = s.repo.LoadAccount(c, accountID, a); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "could not load account: %v", err.Error()) + } s.log.Error().Err(err).Str("id", accountID).Msg("could not load account") - return + return merrors.InternalServerError(s.id, "could not load account: %v", err.Error()) } g := &proto.Group{} - if err = s.loadGroup(groupID, g); err != nil { + if err = s.repo.LoadGroup(c, groupID, g); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "could not load group: %v", err.Error()) + } s.log.Error().Err(err).Str("id", groupID).Msg("could not load group") - return + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } //remove the account from the group if it exists @@ -391,13 +343,13 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest, } a.MemberOf = newGroups - if err = s.writeAccount(a); err != nil { + if err = s.repo.WriteAccount(c, a); err != nil { s.log.Error().Err(err).Interface("account", a).Msg("could not persist account") - return + return merrors.InternalServerError(s.id, "could not persist account: %v", err.Error()) } - if err = s.writeGroup(g); err != nil { + if err = s.repo.WriteGroup(c, g); err != nil { s.log.Error().Err(err).Interface("group", g).Msg("could not persist group") - return + return merrors.InternalServerError(s.id, "could not persist group: %v", err.Error()) } // FIXME update index! // TODO rollback changes when only one of them failed? @@ -408,7 +360,6 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest, // ListMembers implements the GroupsServiceHandler interface func (s Service) ListMembers(c context.Context, in *proto.ListMembersRequest, out *proto.ListMembersResponse) (err error) { - // cleanup ids var groupID string if groupID, err = cleanupID(in.Id); err != nil { @@ -416,16 +367,17 @@ func (s Service) ListMembers(c context.Context, in *proto.ListMembersRequest, ou } g := &proto.Group{} - if err = s.loadGroup(groupID, g); err != nil { + if err = s.repo.LoadGroup(c, groupID, g); err != nil { + if storage.IsNotFoundErr(err) { + return merrors.NotFound(s.id, "group not found: %v", err.Error()) + } s.log.Error().Err(err).Str("id", groupID).Msg("could not load group") - return + return merrors.InternalServerError(s.id, "could not load group: %v", err.Error()) } // TODO only expand accounts if requested // if in.FieldMask ... s.expandMembers(g) - out.Members = g.Members - return } diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index 38181cb9fbd..772210253ec 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -2,10 +2,8 @@ package service import ( "context" - "encoding/json" "errors" - "fmt" - "io/ioutil" + "github.com/owncloud/ocis/accounts/pkg/storage" "os" "path/filepath" "strings" @@ -56,331 +54,292 @@ func New(opts ...Option) (s *Service, err error) { Config: cfg, RoleService: roleService, RoleManager: roleManager, + repo: createMetadataStorage(cfg, logger), } - // build an index if s.index, err = s.buildIndex(); err != nil { return nil, err } - // create default accounts - accountsDir := filepath.Join(cfg.Server.AccountsDataPath, "accounts") - if err = s.createDefaultAccounts(accountsDir); err != nil { - return nil, err - } - if err = s.indexAccounts(accountsDir); err != nil { + if err = s.createDefaultAccounts(); err != nil { return nil, err } - // create default groups - groupsDir := filepath.Join(cfg.Server.AccountsDataPath, "groups") - if err = s.createDefaultGroups(groupsDir); err != nil { - return nil, err - } - if err = s.indexGroups(groupsDir); err != nil { + if err = s.createDefaultGroups(); err != nil { return nil, err } - // TODO watch folders for new records - return } func (s Service) buildIndex() (index bleve.Index, err error) { - indexMapping := bleve.NewIndexMapping() - // keep all symbols in terms to allow exact maching, eg. emails - indexMapping.DefaultAnalyzer = keyword.Name - // TODO don't bother to store fields as we will load the account from disk - - // Reusable mapping for text - standardTextFieldMapping := bleve.NewTextFieldMapping() - standardTextFieldMapping.Analyzer = standard.Name - standardTextFieldMapping.Store = false - - // Reusable mapping for text, uses english stop word removal - simpleTextFieldMapping := bleve.NewTextFieldMapping() - simpleTextFieldMapping.Analyzer = simple.Name - simpleTextFieldMapping.Store = false - - // Reusable mapping for keyword text - keywordFieldMapping := bleve.NewTextFieldMapping() - keywordFieldMapping.Analyzer = keyword.Name - keywordFieldMapping.Store = false - - // Reusable mapping for lowercase text - err = indexMapping.AddCustomAnalyzer("lowercase", - map[string]interface{}{ - "type": custom.Name, - "tokenizer": unicode.Name, - "token_filters": []string{ - lowercase.Name, - }, - }) - if err != nil { - return - } - lowercaseTextFieldMapping := bleve.NewTextFieldMapping() - lowercaseTextFieldMapping.Analyzer = "lowercase" - lowercaseTextFieldMapping.Store = true + indexDir := filepath.Join(s.Config.Server.AccountsDataPath, "index.bleve") + if index, err = bleve.Open(indexDir); err != nil { + if err != bleve.ErrorIndexPathDoesNotExist { + s.log.Error().Err(err).Msg("failed to read index") + return + } - // accounts - accountMapping := bleve.NewDocumentMapping() - indexMapping.AddDocumentMapping("account", accountMapping) + indexMapping := bleve.NewIndexMapping() + // keep all symbols in terms to allow exact maching, eg. emails + indexMapping.DefaultAnalyzer = keyword.Name + // TODO don't bother to store fields as we will load the account from disk - // Text - accountMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping) - accountMapping.AddFieldMappingsAt("description", standardTextFieldMapping) + // Reusable mapping for text + standardTextFieldMapping := bleve.NewTextFieldMapping() + standardTextFieldMapping.Analyzer = standard.Name + standardTextFieldMapping.Store = false - // Lowercase - accountMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping) - accountMapping.AddFieldMappingsAt("preferred_name", lowercaseTextFieldMapping) + // Reusable mapping for text, uses english stop word removal + simpleTextFieldMapping := bleve.NewTextFieldMapping() + simpleTextFieldMapping.Analyzer = simple.Name + simpleTextFieldMapping.Store = false - // Keywords - accountMapping.AddFieldMappingsAt("mail", keywordFieldMapping) + // Reusable mapping for keyword text + keywordFieldMapping := bleve.NewTextFieldMapping() + keywordFieldMapping.Analyzer = keyword.Name + keywordFieldMapping.Store = false - // groups - groupMapping := bleve.NewDocumentMapping() - indexMapping.AddDocumentMapping("group", groupMapping) + // Reusable mapping for lowercase text + err = indexMapping.AddCustomAnalyzer("lowercase", + map[string]interface{}{ + "type": custom.Name, + "tokenizer": unicode.Name, + "token_filters": []string{ + lowercase.Name, + }, + }) + if err != nil { + return + } + lowercaseTextFieldMapping := bleve.NewTextFieldMapping() + lowercaseTextFieldMapping.Analyzer = "lowercase" + lowercaseTextFieldMapping.Store = true - // Text - groupMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping) - groupMapping.AddFieldMappingsAt("description", standardTextFieldMapping) + // accounts + accountMapping := bleve.NewDocumentMapping() + indexMapping.AddDocumentMapping("account", accountMapping) - // Lowercase - groupMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping) + // Text + accountMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping) + accountMapping.AddFieldMappingsAt("description", standardTextFieldMapping) - // Tell blevesearch how to determine the type of the structs that are indexed. - // The referenced field needs to match the struct field exactly and it must be public. - // See pkg/proto/v0/bleve.go how we wrap the generated Account and Group to add a - // BleveType property which is indexed as `bleve_type` so we can also distinguish the - // documents in the index by querying for that property. - indexMapping.TypeField = "BleveType" + // Lowercase + accountMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping) + accountMapping.AddFieldMappingsAt("preferred_name", lowercaseTextFieldMapping) - indexDir := filepath.Join(s.Config.Server.AccountsDataPath, "index.bleve") - // for now recreate index on every start - if err = os.RemoveAll(indexDir); err != nil { - return - } - if index, err = bleve.New(indexDir, indexMapping); err != nil { - return nil, err + // Keywords + accountMapping.AddFieldMappingsAt("mail", keywordFieldMapping) + + // groups + groupMapping := bleve.NewDocumentMapping() + indexMapping.AddDocumentMapping("group", groupMapping) + + // Text + groupMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping) + groupMapping.AddFieldMappingsAt("description", standardTextFieldMapping) + + // Lowercase + groupMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping) + + // Tell blevesearch how to determine the type of the structs that are indexed. + // The referenced field needs to match the struct field exactly and it must be public. + // See pkg/proto/v0/bleve.go how we wrap the generated Account and Group to add a + // BleveType property which is indexed as `bleve_type` so we can also distinguish the + // documents in the index by querying for that property. + indexMapping.TypeField = "BleveType" + + // for now recreate index on every start + if err = os.RemoveAll(indexDir); err != nil { + return + } + if index, err = bleve.New(indexDir, indexMapping); err != nil { + return nil, err + } } return } -func (s Service) createDefaultAccounts(accountsDir string) (err error) { - // check if accounts exist - var fi os.FileInfo - if fi, err = os.Stat(accountsDir); err != nil { - if os.IsNotExist(err) { - // create accounts directory - if err = os.MkdirAll(accountsDir, 0700); err != nil { - return - } - // create default accounts - accounts := []proto.Account{ - { - Id: "4c510ada-c86b-4815-8820-42cdf82c3d51", - PreferredName: "einstein", - OnPremisesSamAccountName: "einstein", - Mail: "einstein@example.org", - DisplayName: "Albert Einstein", - UidNumber: 20000, - GidNumber: 30000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=35210$sa1u5Pmfo4cr23Vw$RJNGElaDB1D3xorWkfTEGm2Ko.o2QL3E0cimKx23MNxVWVFSkUUeRoC7FqC4RzYDNQBD6cKzovTEaDD.8TDkD.", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users - {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0"}, // sailing-lovers - {Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f"}, // violin-haters - {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers - }, - }, - { - Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - PreferredName: "marie", - OnPremisesSamAccountName: "marie", - Mail: "marie@example.org", - DisplayName: "Marie Curie", - UidNumber: 20001, - GidNumber: 30000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=81434$sa1u5Pmfo4cr23Vw$W78cyL884GmuvDpxYPvSRBVzEj02T5QhTTcI8Dv4IKvMooDFGv4bwaWMkH9HfJ0wgpEBW7Lp.4Cad0xE/MYSg1", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users - {Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a"}, // radium-lovers - {Id: "cedc21aa-4072-4614-8676-fa9165f598ff"}, // polonium-lovers - {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers - }, - }, - { - Id: "932b4540-8d16-481e-8ef4-588e4b6b151c", - PreferredName: "richard", - OnPremisesSamAccountName: "richard", - Mail: "richard@example.org", - DisplayName: "Richard Feynman", - UidNumber: 20002, - GidNumber: 30000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=5524$sa1u5Pmfo4cr23Vw$58bQVL/JeUlwM0RY21YKAFMvKvwKLLysGllYXox.vwKT5dHMwdzJjCxwTDMnB2o2pwexC8o/iOXyP2zrhALS40", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users - {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a"}, // quantum-lovers - {Id: "167cbee2-0518-455a-bfb2-031fe0621e5d"}, // philosophy-haters - {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers - }, - }, - // admin user(s) - { - Id: "058bff95-6708-4fe5-91e4-9ea3d377588b", - PreferredName: "moss", - OnPremisesSamAccountName: "moss", - Mail: "moss@example.org", - DisplayName: "Maurice Moss", - UidNumber: 20003, - GidNumber: 30000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=47068$lhw6odzXW0LTk/ao$GgxS.pIgP8jawLJBAiyNor2FrWzrULF95PwspRkli2W3VF.4HEwTYlQfRXbNQBMjNCEcEYlgZo3a.kRz2k2N0/", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users - }, - }, - // technical users for kopano and reva - { - Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf", - PreferredName: "konnectd", - OnPremisesSamAccountName: "konnectd", - Mail: "idp@example.org", - DisplayName: "Kopano Konnectd", - UidNumber: 10000, - GidNumber: 15000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=9746$sa1u5Pmfo4cr23Vw$2hnwpkTvUkWX0v6mh8Aw1pbzEXa9EUJzmrey4g2W/8arwWCwhteqU//3aWnA3S0d5T21fOKYteoqlsN1IbTcN.", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers - }, - }, - { - Id: "bc596f3c-c955-4328-80a0-60d018b4ad57", - PreferredName: "reva", - OnPremisesSamAccountName: "reva", - Mail: "storage@example.org", - DisplayName: "Reva Inter Operability Platform", - UidNumber: 10001, - GidNumber: 15000, - PasswordProfile: &proto.PasswordProfile{ - Password: "$6$rounds=91087$sa1u5Pmfo4cr23Vw$wPC3BbMTbP/ytlo0p.f99zJifyO70AUCdKIK9hkhwutBKGCirLmZs/MsWAG6xHjVvmnmHN5NoON7FUGv5pPaN.", - }, - AccountEnabled: true, - MemberOf: []*proto.Group{ - {Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers - }, - }, - } - for i := range accounts { - // create account on disk - var bytes []byte - if bytes, err = json.Marshal(&accounts[i]); err != nil { - s.log.Error().Err(err).Interface("account", &accounts[i]).Msg("could not marshal default account") - return - } - path := filepath.Join(accountsDir, accounts[i].Id) - if err = ioutil.WriteFile(path, bytes, 0600); err != nil { - accounts[i].PasswordProfile.Password = "***REMOVED***" - s.log.Error().Err(err).Str("path", path).Interface("account", &accounts[i]).Msg("could not persist default account") - return - } - } - - // set role for admin users and regular users - assignRoleToUser("058bff95-6708-4fe5-91e4-9ea3d377588b", settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log) - for _, accountID := range []string{ - "058bff95-6708-4fe5-91e4-9ea3d377588b", //moss - } { - assignRoleToUser(accountID, settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log) - } - for _, accountID := range []string{ - "4c510ada-c86b-4815-8820-42cdf82c3d51", //einstein - "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", //marie - "932b4540-8d16-481e-8ef4-588e4b6b151c", //richard - } { - assignRoleToUser(accountID, settings_svc.BundleUUIDRoleUser, s.RoleService, s.log) - } +func (s Service) createDefaultAccounts() (err error) { + accounts := []proto.Account{ + { + Id: "4c510ada-c86b-4815-8820-42cdf82c3d51", + PreferredName: "einstein", + OnPremisesSamAccountName: "einstein", + Mail: "einstein@example.org", + DisplayName: "Albert Einstein", + UidNumber: 20000, + GidNumber: 30000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=35210$sa1u5Pmfo4cr23Vw$RJNGElaDB1D3xorWkfTEGm2Ko.o2QL3E0cimKx23MNxVWVFSkUUeRoC7FqC4RzYDNQBD6cKzovTEaDD.8TDkD.", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users + {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0"}, // sailing-lovers + {Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f"}, // violin-haters + {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers + }, + }, + { + Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + PreferredName: "marie", + OnPremisesSamAccountName: "marie", + Mail: "marie@example.org", + DisplayName: "Marie Curie", + UidNumber: 20001, + GidNumber: 30000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=81434$sa1u5Pmfo4cr23Vw$W78cyL884GmuvDpxYPvSRBVzEj02T5QhTTcI8Dv4IKvMooDFGv4bwaWMkH9HfJ0wgpEBW7Lp.4Cad0xE/MYSg1", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users + {Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a"}, // radium-lovers + {Id: "cedc21aa-4072-4614-8676-fa9165f598ff"}, // polonium-lovers + {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers + }, + }, + { + Id: "932b4540-8d16-481e-8ef4-588e4b6b151c", + PreferredName: "richard", + OnPremisesSamAccountName: "richard", + Mail: "richard@example.org", + DisplayName: "Richard Feynman", + UidNumber: 20002, + GidNumber: 30000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=5524$sa1u5Pmfo4cr23Vw$58bQVL/JeUlwM0RY21YKAFMvKvwKLLysGllYXox.vwKT5dHMwdzJjCxwTDMnB2o2pwexC8o/iOXyP2zrhALS40", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users + {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a"}, // quantum-lovers + {Id: "167cbee2-0518-455a-bfb2-031fe0621e5d"}, // philosophy-haters + {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers + }, + }, + // admin user(s) + { + Id: "058bff95-6708-4fe5-91e4-9ea3d377588b", + PreferredName: "moss", + OnPremisesSamAccountName: "moss", + Mail: "moss@example.org", + DisplayName: "Maurice Moss", + UidNumber: 20003, + GidNumber: 30000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=47068$lhw6odzXW0LTk/ao$GgxS.pIgP8jawLJBAiyNor2FrWzrULF95PwspRkli2W3VF.4HEwTYlQfRXbNQBMjNCEcEYlgZo3a.kRz2k2N0/", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users + }, + }, + // technical users for kopano and reva + { + Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf", + PreferredName: "konnectd", + OnPremisesSamAccountName: "konnectd", + Mail: "idp@example.org", + DisplayName: "Kopano Konnectd", + UidNumber: 10000, + GidNumber: 15000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=9746$sa1u5Pmfo4cr23Vw$2hnwpkTvUkWX0v6mh8Aw1pbzEXa9EUJzmrey4g2W/8arwWCwhteqU//3aWnA3S0d5T21fOKYteoqlsN1IbTcN.", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers + }, + }, + { + Id: "bc596f3c-c955-4328-80a0-60d018b4ad57", + PreferredName: "reva", + OnPremisesSamAccountName: "reva", + Mail: "storage@example.org", + DisplayName: "Reva Inter Operability Platform", + UidNumber: 10001, + GidNumber: 15000, + PasswordProfile: &proto.PasswordProfile{ + Password: "$6$rounds=91087$sa1u5Pmfo4cr23Vw$wPC3BbMTbP/ytlo0p.f99zJifyO70AUCdKIK9hkhwutBKGCirLmZs/MsWAG6xHjVvmnmHN5NoON7FUGv5pPaN.", + }, + AccountEnabled: true, + MemberOf: []*proto.Group{ + {Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers + }, + }, + } + for i := range accounts { + if err := s.repo.WriteAccount(context.Background(), &accounts[i]); err != nil { + return err } - } else if !fi.IsDir() { - return fmt.Errorf("%s is not a directory", accountsDir) + + if err := s.indexAccount(accounts[i].Id); err != nil { + return err + } + } + + // set role for admin users and regular users + assignRoleToUser("058bff95-6708-4fe5-91e4-9ea3d377588b", settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log) + for _, accountID := range []string{ + "058bff95-6708-4fe5-91e4-9ea3d377588b", //moss + } { + assignRoleToUser(accountID, settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log) + } + for _, accountID := range []string{ + "4c510ada-c86b-4815-8820-42cdf82c3d51", //einstein + "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", //marie + "932b4540-8d16-481e-8ef4-588e4b6b151c", //richard + } { + assignRoleToUser(accountID, settings_svc.BundleUUIDRoleUser, s.RoleService, s.log) } return nil } -func (s Service) createDefaultGroups(groupsDir string) (err error) { - // check if groups exist - var fi os.FileInfo - if fi, err = os.Stat(groupsDir); err != nil { - if os.IsNotExist(err) { - // create accounts directory - if err = os.MkdirAll(groupsDir, 0700); err != nil { - return - } - // create default accounts - groups := []proto.Group{ - {Id: "34f38767-c937-4eb6-b847-1c175829a2a0", GidNumber: 15000, OnPremisesSamAccountName: "sysusers", DisplayName: "Technical users", Description: "A group for technical users. They should not show up in sharing dialogs.", Members: []*proto.Account{ - {Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf"}, // konnectd - {Id: "bc596f3c-c955-4328-80a0-60d018b4ad57"}, // reva - }}, - {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa", GidNumber: 30000, OnPremisesSamAccountName: "users", DisplayName: "Users", Description: "A group every normal user belongs to.", Members: []*proto.Account{ - {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein - {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie - {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman - }}, - {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0", GidNumber: 30001, OnPremisesSamAccountName: "sailing-lovers", DisplayName: "Sailing lovers", Members: []*proto.Account{ - {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein - }}, - {Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f", GidNumber: 30002, OnPremisesSamAccountName: "violin-haters", DisplayName: "Violin haters", Members: []*proto.Account{ - {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein - }}, - {Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a", GidNumber: 30003, OnPremisesSamAccountName: "radium-lovers", DisplayName: "Radium lovers", Members: []*proto.Account{ - {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie - }}, - {Id: "cedc21aa-4072-4614-8676-fa9165f598ff", GidNumber: 30004, OnPremisesSamAccountName: "polonium-lovers", DisplayName: "Polonium lovers", Members: []*proto.Account{ - {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie - }}, - {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a", GidNumber: 30005, OnPremisesSamAccountName: "quantum-lovers", DisplayName: "Quantum lovers", Members: []*proto.Account{ - {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman - }}, - {Id: "167cbee2-0518-455a-bfb2-031fe0621e5d", GidNumber: 30006, OnPremisesSamAccountName: "philosophy-haters", DisplayName: "Philosophy haters", Members: []*proto.Account{ - {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman - }}, - {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e", GidNumber: 30007, OnPremisesSamAccountName: "physics-lovers", DisplayName: "Physics lovers", Members: []*proto.Account{ - {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein - {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie - {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman - }}, - } - for i := range groups { - var bytes []byte - if bytes, err = json.Marshal(&groups[i]); err != nil { - s.log.Error().Err(err).Interface("group", &groups[i]).Msg("could not marshal default group") - return - } - path := filepath.Join(groupsDir, groups[i].Id) - if err = ioutil.WriteFile(path, bytes, 0600); err != nil { - s.log.Error().Err(err).Str("path", path).Interface("group", &groups[i]).Msg("could not persist default group") - return - } - } +func (s Service) createDefaultGroups() (err error) { + groups := []proto.Group{ + {Id: "34f38767-c937-4eb6-b847-1c175829a2a0", GidNumber: 15000, OnPremisesSamAccountName: "sysusers", DisplayName: "Technical users", Description: "A group for technical users. They should not show up in sharing dialogs.", Members: []*proto.Account{ + {Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf"}, // konnectd + {Id: "bc596f3c-c955-4328-80a0-60d018b4ad57"}, // reva + }}, + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa", GidNumber: 30000, OnPremisesSamAccountName: "users", DisplayName: "Users", Description: "A group every normal user belongs to.", Members: []*proto.Account{ + {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein + {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie + {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + }}, + {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0", GidNumber: 30001, OnPremisesSamAccountName: "sailing-lovers", DisplayName: "Sailing lovers", Members: []*proto.Account{ + {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein + }}, + {Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f", GidNumber: 30002, OnPremisesSamAccountName: "violin-haters", DisplayName: "Violin haters", Members: []*proto.Account{ + {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein + }}, + {Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a", GidNumber: 30003, OnPremisesSamAccountName: "radium-lovers", DisplayName: "Radium lovers", Members: []*proto.Account{ + {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie + }}, + {Id: "cedc21aa-4072-4614-8676-fa9165f598ff", GidNumber: 30004, OnPremisesSamAccountName: "polonium-lovers", DisplayName: "Polonium lovers", Members: []*proto.Account{ + {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie + }}, + {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a", GidNumber: 30005, OnPremisesSamAccountName: "quantum-lovers", DisplayName: "Quantum lovers", Members: []*proto.Account{ + {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + }}, + {Id: "167cbee2-0518-455a-bfb2-031fe0621e5d", GidNumber: 30006, OnPremisesSamAccountName: "philosophy-haters", DisplayName: "Philosophy haters", Members: []*proto.Account{ + {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + }}, + {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e", GidNumber: 30007, OnPremisesSamAccountName: "physics-lovers", DisplayName: "Physics lovers", Members: []*proto.Account{ + {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein + {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie + {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + }}, + } + for i := range groups { + if err := s.repo.WriteGroup(context.Background(), &groups[i]); err != nil { + return err + } + + if err := s.indexGroup(groups[i].Id); err != nil { + return err } - } else if !fi.IsDir() { - return fmt.Errorf("%s is not a directory", groupsDir) } return nil } @@ -397,6 +356,19 @@ func assignRoleToUser(accountID, roleID string, rs settings.RoleService, logger return true } +func createMetadataStorage(cfg *config.Config, logger log.Logger) storage.Repo { + // for now we detect the used storage implementation based on which storage is configured + // the config with defaults needs to be checked last + if cfg.Repo.Disk.Path != "" { + return storage.NewDiskRepo(cfg, logger) + } + repo, err := storage.NewCS3Repo(cfg) + if err != nil { + logger.Fatal().Err(err).Msg("cs3 storage was configured but failed to start") + } + return repo +} + // Service implements the AccountsServiceHandler interface type Service struct { id string @@ -405,6 +377,7 @@ type Service struct { index bleve.Index RoleService settings.RoleService RoleManager *roles.Manager + repo storage.Repo } func cleanupID(id string) (string, error) { diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go new file mode 100644 index 00000000000..5fce0839c0f --- /dev/null +++ b/accounts/pkg/storage/cs3.go @@ -0,0 +1,287 @@ +package storage + +import ( + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "path" + "strings" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" + "github.com/cs3org/reva/pkg/token/manager/jwt" + "github.com/owncloud/ocis/accounts/pkg/config" + "github.com/owncloud/ocis/accounts/pkg/proto/v0" + "google.golang.org/grpc/metadata" +) + +// CS3Repo provides a cs3 implementation of the Repo interface +type CS3Repo struct { + cfg *config.Config + tm token.Manager + storageProvider provider.ProviderAPIClient + dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol +} + +// NewCS3Repo creates a new cs3 repo +func NewCS3Repo(cfg *config.Config) (Repo, error) { + tokenManager, err := jwt.New(map[string]interface{}{ + "secret": cfg.TokenManager.JWTSecret, + }) + + if err != nil { + return nil, err + } + + client, err := pool.GetStorageProviderServiceClient(cfg.Repo.CS3.ProviderAddr) + if err != nil { + return nil, err + } + + return CS3Repo{ + cfg: cfg, + tm: tokenManager, + storageProvider: client, + dataProvider: dataProviderClient{ + client: http.Client{ + Transport: http.DefaultTransport, + }, + }, + }, nil +} + +// WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id). +func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil { + return err + } + + var by []byte + if by, err = json.Marshal(a); err != nil { + return err + } + + _, err = r.dataProvider.put(r.accountURL(a.Id), bytes.NewReader(by), t) + return err +} + +// LoadAccount loads an account via cs3 by id and writes it to the provided account +func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + resp, err := r.dataProvider.get(r.accountURL(id), t) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusNotFound { + return ¬FoundErr{"account", id} + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if err = resp.Body.Close(); err != nil { + return err + } + return json.Unmarshal(b, &a) +} + +// DeleteAccount deletes an account via cs3 by id +func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + + resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", accountsFolder, id)}, + }, + }) + + if err != nil { + return err + } + + // TODO Handle other error codes? + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + return ¬FoundErr{"account", id} + } + + return nil +} + +// WriteGroup writes a group via cs3 and modifies the provided group (e.g. with a generated id). +func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil { + return err + } + + var by []byte + if by, err = json.Marshal(g); err != nil { + return err + } + + _, err = r.dataProvider.put(r.groupURL(g.Id), bytes.NewReader(by), t) + return err +} + +// LoadGroup loads a group via cs3 by id and writes it to the provided group +func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + resp, err := r.dataProvider.get(r.groupURL(id), t) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusNotFound { + return ¬FoundErr{"group", id} + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if err = resp.Body.Close(); err != nil { + return err + } + return json.Unmarshal(b, &g) +} + +// DeleteGroup deletes a group via cs3 by id +func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + + resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", groupsFolder, id)}, + }, + }) + + if err != nil { + return err + } + + // TODO Handle other error codes? + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + return ¬FoundErr{"group", id} + } + + return err +} + +func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) { + u := &user.User{ + Id: &user.UserId{}, + Groups: []string{}, + } + if r.cfg.ServiceUser.Username != "" { + u.Id.OpaqueId = r.cfg.ServiceUser.UUID + } + return r.tm.MintToken(ctx, u) +} + +func (r CS3Repo) accountURL(id string) string { + return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, accountsFolder, id)) +} + +func (r CS3Repo) groupURL(id string) string { + return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, groupsFolder, id)) +} + +func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error { + var rootPathRef = &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", folder)}, + } + + resp, err := r.storageProvider.Stat(ctx, &provider.StatRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + _, err := r.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + } + + return nil +} + +// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg +func singleJoiningSlash(a, b string) string { + aslash := strings.HasSuffix(a, "/") + bslash := strings.HasPrefix(b, "/") + switch { + case aslash && bslash: + return a + b[1:] + case !aslash && !bslash: + return a + "/" + b + } + return a + b +} + +type dataProviderClient struct { + client http.Client +} + +func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, body) + if err != nil { + return nil, err + } + + req.Header.Add("x-access-token", token) + return d.client.Do(req) +} + +func (d dataProviderClient) get(url string, token string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("x-access-token", token) + return d.client.Do(req) +} diff --git a/accounts/pkg/storage/cs3_test.go b/accounts/pkg/storage/cs3_test.go new file mode 100644 index 00000000000..d788dc7ac38 --- /dev/null +++ b/accounts/pkg/storage/cs3_test.go @@ -0,0 +1,74 @@ +package storage + +// Uncomment to test locally, requires started metadata-storage for now + +//import ( +// "context" +// "github.com/owncloud/ocis/accounts/pkg/config" +// "github.com/owncloud/ocis/accounts/pkg/proto/v0" +// "github.com/stretchr/testify/assert" +// "testing" +//) +// +//var cfg = &config.Config{ +// TokenManager: config.TokenManager{ +// JWTSecret: "Pive-Fumkiu4", +// }, +// Repo: config.Repo{ +// CS3: config.CS3{ +// ProviderAddr: "0.0.0.0:9215", +// DataURL: "http://localhost:9216", +// DataPrefix: "data", +// }, +// }, +//} +// +//func TestCS3Repo_WriteAccount(t *testing.T) { +// r, err := NewCS3Repo("hello", cfg) +// assert.NoError(t, err) +// +// err = r.WriteAccount(context.Background(), &proto.Account{ +// Id: "fefef-egegweg-gegeg", +// AccountEnabled: true, +// DisplayName: "Mike Jones", +// Mail: "mike@example.com", +// }) +// +// assert.NoError(t, err) +//} +// +//func TestCS3Repo_LoadAccount(t *testing.T) { +// r, err := NewCS3Repo("hello", cfg) +// assert.NoError(t, err) +// +// err = r.WriteAccount(context.Background(), &proto.Account{ +// Id: "fefef-egegweg-gegeg", +// AccountEnabled: true, +// DisplayName: "Mike Jones", +// Mail: "mike@example.com", +// }) +// +// acc := &proto.Account{} +// err = r.LoadAccount(context.Background(), "fefef-egegweg-gegeg", acc) +// +// assert.NoError(t, err) +// assert.Equal(t, "fefef-egegweg-gegeg", acc.Id) +// assert.Equal(t, "Mike Jones", acc.DisplayName) +// assert.Equal(t, "mike@example.com", acc.Mail) +//} +// +//func TestCS3Repo_DeleteAccount(t *testing.T) { +// r, err := NewCS3Repo("hello", cfg) +// assert.NoError(t, err) +// +// err = r.WriteAccount(context.Background(), &proto.Account{ +// Id: "delete-me-id", +// AccountEnabled: true, +// DisplayName: "Mike Jones", +// Mail: "mike@example.com", +// }) +// +// err = r.DeleteAccount(context.Background(), "delete-me-id") +// +// assert.NoError(t, err) +//} diff --git a/accounts/pkg/storage/disk.go b/accounts/pkg/storage/disk.go new file mode 100644 index 00000000000..3490424a6de --- /dev/null +++ b/accounts/pkg/storage/disk.go @@ -0,0 +1,168 @@ +package storage + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/owncloud/ocis/accounts/pkg/config" + "github.com/owncloud/ocis/accounts/pkg/proto/v0" + olog "github.com/owncloud/ocis/ocis-pkg/log" +) + +var groupLock sync.Mutex + +// DiskRepo provides a local filesystem implementation of the Repo interface +type DiskRepo struct { + cfg *config.Config + log olog.Logger +} + +// NewDiskRepo creates a new disk repo +func NewDiskRepo(cfg *config.Config, log olog.Logger) DiskRepo { + paths := []string{ + filepath.Join(cfg.Repo.Disk.Path, accountsFolder), + filepath.Join(cfg.Repo.Disk.Path, groupsFolder), + } + for i := range paths { + if _, err := os.Stat(paths[i]); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(paths[i], 0700); err != nil { + log.Fatal().Err(err).Msgf("could not create data folder %v", paths[i]) + } + } + } + } + return DiskRepo{ + cfg: cfg, + log: log, + } +} + +// WriteAccount to the local filesystem +func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error) { + // leave only the group id + r.deflateMemberOf(a) + + var bytes []byte + if bytes, err = json.Marshal(a); err != nil { + return err + } + + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, a.Id) + return ioutil.WriteFile(path, bytes, 0600) +} + +// LoadAccount from the local filesystem +func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id) + var data []byte + if data, err = ioutil.ReadFile(path); err != nil { + if os.IsNotExist(err) { + err = ¬FoundErr{"account", id} + } + return + } + + return json.Unmarshal(data, a) +} + +// DeleteAccount from the local filesystem +func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id) + if err = os.Remove(path); err != nil { + if os.IsNotExist(err) { + err = ¬FoundErr{"account", id} + } + } + + return +} + +// WriteGroup to the local filesystem +func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { + // leave only the member id + r.deflateMembers(g) + + var bytes []byte + if bytes, err = json.Marshal(g); err != nil { + return err + } + + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, g.Id) + + groupLock.Lock() + defer groupLock.Unlock() + + return ioutil.WriteFile(path, bytes, 0600) +} + +// LoadGroup from the local filesystem +func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id) + + groupLock.Lock() + defer groupLock.Unlock() + var data []byte + if data, err = ioutil.ReadFile(path); err != nil { + if os.IsNotExist(err) { + err = ¬FoundErr{"group", id} + } + + return + } + + return json.Unmarshal(data, g) +} + +// DeleteGroup from the local filesystem +func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id) + if err = os.Remove(path); err != nil { + if os.IsNotExist(err) { + err = ¬FoundErr{"account", id} + } + } + + return nil + + //r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group") + //return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error()) +} + +// deflateMemberOf replaces the groups of a user with an instance that only contains the id +func (r DiskRepo) deflateMemberOf(a *proto.Account) { + if a == nil { + return + } + var deflated []*proto.Group + for i := range a.MemberOf { + if a.MemberOf[i].Id != "" { + deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id}) + } else { + // TODO fetch and use an id when group only has a name but no id + r.log.Error().Str("id", a.Id).Interface("group", a.MemberOf[i]).Msg("resolving groups by name is not implemented yet") + } + } + a.MemberOf = deflated +} + +// deflateMembers replaces the users of a group with an instance that only contains the id +func (r DiskRepo) deflateMembers(g *proto.Group) { + if g == nil { + return + } + var deflated []*proto.Account + for i := range g.Members { + if g.Members[i].Id != "" { + deflated = append(deflated, &proto.Account{Id: g.Members[i].Id}) + } else { + // TODO fetch and use an id when group only has a name but no id + r.log.Error().Str("id", g.Id).Interface("account", g.Members[i]).Msg("resolving members by name is not implemented yet") + } + } + g.Members = deflated +} diff --git a/accounts/pkg/storage/errors.go b/accounts/pkg/storage/errors.go new file mode 100644 index 00000000000..3dfa306e56a --- /dev/null +++ b/accounts/pkg/storage/errors.go @@ -0,0 +1,19 @@ +package storage + +import ( + "fmt" +) + +type notFoundErr struct { + typ, id string +} + +func (e *notFoundErr) Error() string { + return fmt.Sprintf("%s with id %s not found", e.typ, e.id) +} + +// IsNotFoundErr can be returned by repo Load and Delete operations +func IsNotFoundErr(e error) bool { + _, ok := e.(*notFoundErr) + return ok +} diff --git a/accounts/pkg/storage/repo.go b/accounts/pkg/storage/repo.go new file mode 100644 index 00000000000..27f278876aa --- /dev/null +++ b/accounts/pkg/storage/repo.go @@ -0,0 +1,21 @@ +package storage + +import ( + "context" + "github.com/owncloud/ocis/accounts/pkg/proto/v0" +) + +const ( + accountsFolder = "accounts" + groupsFolder = "groups" +) + +// Repo defines the storage operations +type Repo interface { + WriteAccount(ctx context.Context, a *proto.Account) (err error) + LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) + DeleteAccount(ctx context.Context, id string) (err error) + WriteGroup(ctx context.Context, g *proto.Group) (err error) + LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) + DeleteGroup(ctx context.Context, id string) (err error) +} diff --git a/ocis/pkg/command/revastoragemetadata.go b/ocis/pkg/command/revastoragemetadata.go new file mode 100644 index 00000000000..61c7af1125f --- /dev/null +++ b/ocis/pkg/command/revastoragemetadata.go @@ -0,0 +1,49 @@ +package command + +import ( + "github.com/micro/cli/v2" + "github.com/owncloud/ocis/ocis-reva/pkg/command" + svcconfig "github.com/owncloud/ocis/ocis-reva/pkg/config" + "github.com/owncloud/ocis/ocis-reva/pkg/flagset" + "github.com/owncloud/ocis/ocis/pkg/config" + "github.com/owncloud/ocis/ocis/pkg/register" +) + +// RevaStorageMetadataCommand is the entrypoint for the reva-storage-metadata command. +func RevaStorageMetadataCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "reva-storage-metadata", + Usage: "Start reva storage service for metadata mount", + Category: "Extensions", + Flags: flagset.StorageMetadata(cfg.Reva), + Action: func(c *cli.Context) error { + revaStorageMetadataCommand := command.StorageMetadata(configureRevaStorageMetadata(cfg)) + + if err := revaStorageMetadataCommand.Before(c); err != nil { + return err + } + + return cli.HandleAction(revaStorageMetadataCommand.Action, c) + }, + } +} + +func configureRevaStorageMetadata(cfg *config.Config) *svcconfig.Config { + cfg.Reva.Log.Level = cfg.Log.Level + cfg.Reva.Log.Pretty = cfg.Log.Pretty + cfg.Reva.Log.Color = cfg.Log.Color + + if cfg.Tracing.Enabled { + cfg.Reva.Tracing.Enabled = cfg.Tracing.Enabled + cfg.Reva.Tracing.Type = cfg.Tracing.Type + cfg.Reva.Tracing.Endpoint = cfg.Tracing.Endpoint + cfg.Reva.Tracing.Collector = cfg.Tracing.Collector + cfg.Reva.Tracing.Service = cfg.Tracing.Service + } + + return cfg.Reva +} + +func init() { + register.AddCommand(RevaStorageMetadataCommand) +} diff --git a/ocis/pkg/command/run.go b/ocis/pkg/command/run.go index 51f62d1c997..47aa12dcaa4 100644 --- a/ocis/pkg/command/run.go +++ b/ocis/pkg/command/run.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "github.com/owncloud/ocis/ocis/pkg/runtime" "log" "net" "net/rpc" @@ -11,7 +12,6 @@ import ( "github.com/owncloud/ocis/ocis/pkg/config" "github.com/owncloud/ocis/ocis/pkg/register" - "github.com/refs/pman/pkg/process" ) // RunCommand is the entrypoint for the run command. @@ -40,13 +40,7 @@ func RunCommand(cfg *config.Config) *cli.Command { log.Fatal("dialing:", err) } - proc := process.NewProcEntry(os.Args[2], os.Environ(), []string{os.Args[2]}...) - var res int - - if err := client.Call("Service.Start", proc, &res); err != nil { - log.Fatal(err) - } - + res := runtime.RunService(client, os.Args[2]) fmt.Println(res) return nil }, diff --git a/ocis/pkg/command/server.go b/ocis/pkg/command/server.go index 6d22e71f914..20927e3f94f 100644 --- a/ocis/pkg/command/server.go +++ b/ocis/pkg/command/server.go @@ -33,31 +33,7 @@ func Server(cfg *config.Config) *cli.Command { } r := runtime.New() - // TODO temporary service startup selection. Should go away and the runtime should take care of it. - return r.Start(append([]string{ - "accounts", - "settings", - "konnectd", - "proxy", - "ocs", - "phoenix", - "glauth", - "webdav", - "store", - "thumbnails", - "reva-frontend", - "reva-gateway", - "reva-users", - "reva-auth-basic", - "reva-auth-bearer", - "reva-storage-home", - "reva-storage-home-data", - "reva-storage-eos", - "reva-storage-eos-data", - "reva-storage-oc", - "reva-storage-oc-data", - "reva-storage-public-link", - }, runtime.MicroServices...)...) + return r.Start() }, } } diff --git a/ocis/pkg/runtime/command.go b/ocis/pkg/runtime/command.go deleted file mode 100644 index 0125f048f4e..00000000000 --- a/ocis/pkg/runtime/command.go +++ /dev/null @@ -1,19 +0,0 @@ -package runtime - -import ( - "github.com/micro/cli/v2" -) - -// Command adds micro runtime commands to the cli app -func Command(app *cli.App) *cli.Command { - command := cli.Command{ - Name: "micro", - Description: "starts the go-micro runtime services", - Category: "Micro", - Action: func(c *cli.Context) error { - runtime := New() - return runtime.Start() - }, - } - return &command -} diff --git a/ocis/pkg/runtime/runtime.go b/ocis/pkg/runtime/runtime.go index 7ea0a833933..0733b8ee0e7 100644 --- a/ocis/pkg/runtime/runtime.go +++ b/ocis/pkg/runtime/runtime.go @@ -50,7 +50,7 @@ var ( "reva-storage-oc", "reva-storage-oc-data", "reva-storage-public-link", - "accounts", + "reva-storage-metadata", "glauth", "konnectd", "thumbnails", @@ -58,6 +58,7 @@ var ( // There seem to be a race condition when reva-sharing needs to read the sharing.json file and the parent folder is not present. dependants = []string{ + "accounts", "reva-sharing", } @@ -74,13 +75,13 @@ func New() Runtime { } // Start rpc runtime -func (r *Runtime) Start(services ...string) error { - go r.Launch(services) +func (r *Runtime) Start() error { + go r.Launch() return service.Start() } // Launch ocis default ocis extensions. -func (r *Runtime) Launch(services []string) { +func (r *Runtime) Launch() { var client *rpc.Client var err error var try int @@ -100,32 +101,37 @@ func (r *Runtime) Launch(services []string) { } OUT: - for _, v := range services { - args := process.NewProcEntry(v, os.Environ(), []string{v}...) - var reply int + for _, v := range MicroServices { + RunService(client, v) + } - if err := client.Call("Service.Start", args, &reply); err != nil { - golog.Fatal(err) - } + for _, v := range Extensions { + RunService(client, v) } - // TODO(refs) this should disappear and tackled at the runtime (pman) level. - // see https://github.com/cs3org/reva/issues/795 for race condition. - // dependants might not be needed on a ocis_simple build, therefore - // it should not be started under these circumstances. - if len(services) >= len(Extensions) { // it will not run for ocis_simple builds. + if len(dependants) > 0 { + // TODO(refs) this should disappear and tackled at the runtime (pman) level. + // see https://github.com/cs3org/reva/issues/795 for race condition. + // dependants might not be needed on a ocis_simple build, therefore + // it should not be started under these circumstances. time.Sleep(2 * time.Second) for _, v := range dependants { - args := process.NewProcEntry(v, os.Environ(), []string{v}...) - var reply int - - if err := client.Call("Service.Start", args, &reply); err != nil { - golog.Fatal(err) - } + RunService(client, v) } } } +// RunService sends a Service.Start command with the given service name to pman +func RunService(client *rpc.Client, service string) int { + args := process.NewProcEntry(service, os.Environ(), []string{service}...) + + var reply int + if err := client.Call("Service.Start", args, &reply); err != nil { + golog.Fatal(err) + } + return reply +} + // AddMicroPlatform adds the micro subcommands to the cli app func AddMicroPlatform(app *cli.App) { setDefaults() diff --git a/ocs/go.mod b/ocs/go.mod index 720d59a48b9..f9e941ae415 100644 --- a/ocs/go.mod +++ b/ocs/go.mod @@ -15,6 +15,7 @@ require ( github.com/micro/cli/v2 v2.1.2 github.com/micro/go-micro/v2 v2.9.1 github.com/oklog/run v1.1.0 + github.com/olekukonko/tablewriter v0.0.4 github.com/openzipkin/zipkin-go v0.2.2 github.com/owncloud/ocis/accounts v0.0.0-20200918125107-fcca9faa81c8 github.com/owncloud/ocis/ocis-pkg v0.0.0-20200918114005-1a0ddd2190ee diff --git a/ocs/go.sum b/ocs/go.sum index 77b0bb99dd8..9a7f7dac78a 100644 --- a/ocs/go.sum +++ b/ocs/go.sum @@ -1927,6 +1927,7 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200421042724-cfa8b22178d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200427214658-4697a2867c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/ocs/pkg/server/http/svc_test.go b/ocs/pkg/server/http/svc_test.go index b18dd98ce70..5dc429038a9 100644 --- a/ocs/pkg/server/http/svc_test.go +++ b/ocs/pkg/server/http/svc_test.go @@ -249,6 +249,11 @@ func init() { Server: accountsCfg.Server{ AccountsDataPath: dataPath, }, + Repo: accountsCfg.Repo{ + Disk: accountsCfg.Disk{ + Path: dataPath, + }, + }, Log: accountsCfg.Log{ Level: "info", Pretty: true,