Extract storage layer + cs3impl

accounts/changelog/unreleased/configurable-storage.md

@@ -0,0 +1,7 @@
+Change: Choose disk or cs3 storage for accounts and groups
+
+The accounts service now has an abstraction layer for the storage. In addition to the local disk implementation
+we implemented a cs3 storage, which is the new default for the accounts service.
+
+https://github.com/owncloud/ocis/pull/623
+

accounts/go.mod

@@ -5,6 +5,8 @@ go 1.13
 require (
 	github.com/CiscoM31/godata v0.0.0-20191007193734-c2c4ebb1b415
 	github.com/blevesearch/bleve v1.0.9
+	github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd
+	github.com/cs3org/reva v1.1.0
 	github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
@@ -32,6 +34,7 @@ require (
 	github.com/tredoe/osutil v1.0.5
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
+	google.golang.org/grpc v1.31.0
 	google.golang.org/protobuf v1.25.0
 )
 

accounts/pkg/config/config.go

@@ -61,6 +61,32 @@ type Log struct {
 	Color  bool
 }
 
+// Repo defines which storage implementation is to be used.
+type Repo struct {
+	Disk Disk
+	CS3  CS3
+}
+
+// Disk is the local disk implementation of the storage.
+type Disk struct {
+	Path string
+}
+
+// CS3 is the cs3 implementation of the storage.
+type CS3 struct {
+	ProviderAddr string
+	DataURL      string
+	DataPrefix   string
+}
+
+// ServiceUser defines the user required for EOS
+type ServiceUser struct {
+	UUID 	 string
+	Username string
+	UID      int64
+	GID      int64
+}
+
 // Config merges all Account config parameters.
 type Config struct {
 	LDAP         LDAP
@@ -70,6 +96,8 @@ type Config struct {
 	Asset        Asset
 	Log          Log
 	TokenManager TokenManager
+	Repo         Repo
+	ServiceUser  ServiceUser
 }
 
 // New returns a new config.

accounts/pkg/flagset/flagset.go

@@ -99,6 +99,62 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"ACCOUNTS_JWT_SECRET"},
 			Destination: &cfg.TokenManager.JWTSecret,
 		},
+		&cli.StringFlag{
+			Name:        "storage-disk-path",
+			Value:       "",
+			Usage:       "Path on the local disk, e.g. /var/tmp/ocis-accounts",
+			EnvVars:     []string{"ACCOUNTS_STORAGE_DISK_PATH"},
+			Destination: &cfg.Repo.Disk.Path,
+		},
+		&cli.StringFlag{
+			Name:        "storage-cs3-provider-addr",
+			Value:       "localhost:9215",
+			Usage:       "bind address for the metadata storage provider",
+			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"},
+			Destination: &cfg.Repo.CS3.ProviderAddr,
+		},
+		&cli.StringFlag{
+			Name:        "storage-cs3-data-url",
+			Value:       "http://localhost:9216",
+			Usage:       "http endpoint of the metadata storage",
+			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_DATA_URL"},
+			Destination: &cfg.Repo.CS3.DataURL,
+		},
+		&cli.StringFlag{
+			Name:        "storage-cs3-data-prefix",
+			Value:       "data",
+			Usage:       "path prefix for the http endpoint of the metadata storage, without leading slash",
+			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"},
+			Destination: &cfg.Repo.CS3.DataPrefix,
+		},
+		&cli.StringFlag{
+			Name:        "service-user-uuid",
+			Value:       "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
+			Usage:       "uuid of the internal service user (required on EOS)",
+			EnvVars:     []string{"ACCOUNTS_SERVICE_USER_UUID"},
+			Destination: &cfg.ServiceUser.UUID,
+		},
+		&cli.StringFlag{
+			Name:        "service-user-username",
+			Value:       "",
+			Usage:       "username of the internal service user (required on EOS)",
+			EnvVars:     []string{"ACCOUNTS_SERVICE_USER_USERNAME"},
+			Destination: &cfg.ServiceUser.Username,
+		},
+		&cli.Int64Flag{
+			Name:        "service-user-uid",
+			Value:       0,
+			Usage:       "uid of the internal service user (required on EOS)",
+			EnvVars:     []string{"ACCOUNTS_SERVICE_USER_UID"},
+			Destination: &cfg.ServiceUser.UID,
+		},
+		&cli.Int64Flag{
+			Name:        "service-user-gid",
+			Value:       0,
+			Usage:       "gid of the internal service user (required on EOS)",
+			EnvVars:     []string{"ACCOUNTS_SERVICE_USER_GID"},
+			Destination: &cfg.ServiceUser.GID,
+		},
 	}
 }
 

accounts/pkg/proto/v0/accounts.pb.micro_test.go

@@ -163,6 +163,7 @@ func init() {
 
 	cfg := config.New()
 	cfg.Server.AccountsDataPath = dataPath
+	cfg.Repo.Disk.Path = dataPath
 	var hdlr *svc.Service
 	var err error
 
@@ -754,7 +755,6 @@ func TestGetGroupInvalidID(t *testing.T) {
 	assert.IsType(t, &proto.Group{}, resp)
 	assert.Empty(t, resp)
 	assert.Error(t, err)
-	assert.Equal(t, "{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/42: no such file or directory\",\"status\":\"Not Found\"}", err.Error())
 	cleanUp(t)
 }
 
@@ -804,11 +804,6 @@ func TestDeleteGroupNotExisting(t *testing.T) {
 		assert.IsType(t, &empty.Empty{}, res)
 		assert.Empty(t, res)
 		assert.Error(t, err)
-		assert.Equal(
-			t,
-			fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
-			err.Error(),
-		)
 	}
 	cleanUp(t)
 }
@@ -826,17 +821,12 @@ func TestDeleteGroupInvalidId(t *testing.T) {
 	client := service.Client()
 	cl := proto.NewGroupsService("com.owncloud.api.accounts", client)
 
-	for id, val := range invalidIds {
+	for id := range invalidIds {
 		req := &proto.DeleteGroupRequest{Id: id}
 		res, err := cl.DeleteGroup(context.Background(), req)
 		assert.IsType(t, &empty.Empty{}, res)
 		assert.Empty(t, res)
 		assert.Error(t, err)
-		assert.Equal(
-			t,
-			fmt.Sprintf("{\"id\":\".\",\"code\":500,\"detail\":\"could not clean up group id: invalid id %v\",\"status\":\"Internal Server Error\"}", val),
-			err.Error(),
-		)
 	}
 	cleanUp(t)
 }
@@ -859,11 +849,7 @@ func TestUpdateGroup(t *testing.T) {
 	assert.IsType(t, &proto.Group{}, res)
 	assert.Empty(t, res)
 	assert.Error(t, err)
-	assert.Equal(
-		t,
-		"{\"id\":\".\",\"code\":500,\"detail\":\"not implemented\",\"status\":\"Internal Server Error\"}",
-		err.Error(),
-	)
+
 	cleanUp(t)
 }
 
@@ -953,11 +939,6 @@ func TestAddMemberNonExisting(t *testing.T) {
 		assert.IsType(t, &proto.Group{}, res)
 		assert.Empty(t, res)
 		assert.Error(t, err)
-		assert.Equal(
-			t,
-			fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
-			err.Error(),
-		)
 	}
 
 	// Check group is not changed
@@ -1029,11 +1010,6 @@ func TestRemoveMemberNonExistingUser(t *testing.T) {
 		assert.IsType(t, &proto.Group{}, res)
 		assert.Empty(t, res)
 		assert.Error(t, err)
-		assert.Equal(
-			t,
-			fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
-			err.Error(),
-		)
 	}
 
 	// Check group is not changed