storage-users: Add config option for graceful shutdown timeout

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d
-	github.com/cs3org/reva/v2 v2.15.0
+	github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9
 	github.com/disintegration/imaging v1.6.2
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
 	github.com/egirna/icap-client v0.1.1

go.sum

@@ -625,8 +625,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
 github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
 github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc=
 github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA=
-github.com/cs3org/reva/v2 v2.15.0 h1:saU2Heig/HswkNdDHh2Jttmhsn0nfTkvaYbXUspcNOM=
-github.com/cs3org/reva/v2 v2.15.0/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w=
+github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9 h1:ycV7H1siLmMiRmc9kaS0WonysHYUT7irvH4FDDAghqQ=
+github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

services/storage-users/README.md

@@ -6,6 +6,18 @@
 
 Starting with ocis version 3.0.0, the default backend for metadata switched to messagepack. If the setting `STORAGE_USERS_OCIS_METADATA_BACKEND` has not been defined manually, the backend will be migrated to `messagepack` automatically. Though still possible to manually configure `xattrs`, this setting should not be used anymore as it will be removed in a later version.
 
+## Graceful Shutdown
+
+Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service.
+
+IMPORTANT: The graceful shtutdown period is only applicapable if the `storage-users` service runs as stand-alone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a stand alone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that the instances configured must be able to communicate on the same network. 
+
+When hard stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written down to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented:
+
+* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing down necessary data. This delay can be necessary if there is a lot of data to be written and/or if storage access/thruput is slow. Though it should not occur but in case you would recieve an error log entry because of not being able to write down data in time, you must increase the default value.
+
+* If a shutdown error has been logged, the command line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. 
+
 ## CLI Commands
 
 ### Manage Unfinished Uploads

services/storage-users/pkg/config/config.go

@@ -21,7 +21,8 @@ type Config struct {
 	TokenManager *TokenManager `yaml:"token_manager"`
 	Reva         *shared.Reva  `yaml:"reva"`
 
-	SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."`
+	SkipUserGroupsInToken   bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."`
+	GracefulShutdownTimeout int  `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details."`
 
 	Driver            string            `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."`
 	Drivers           Drivers           `yaml:"drivers"`

services/storage-users/pkg/config/defaults/defaultconfig.go

@@ -41,12 +41,13 @@ func DefaultConfig() *config.Config {
 		Service: config.Service{
 			Name: "storage-users",
 		},
-		Reva:             shared.DefaultRevaConfig(),
-		DataServerURL:    "http://localhost:9158/data",
-		DataGatewayURL:   "https://localhost:9200/data",
-		TransferExpires:  86400,
-		UploadExpiration: 24 * 60 * 60,
-		Driver:           "ocis",
+		Reva:                    shared.DefaultRevaConfig(),
+		DataServerURL:           "http://localhost:9158/data",
+		DataGatewayURL:          "https://localhost:9200/data",
+		TransferExpires:         86400,
+		UploadExpiration:        24 * 60 * 60,
+		GracefulShutdownTimeout: 30,
+		Driver:                  "ocis",
 		Drivers: config.Drivers{
 			OwnCloudSQL: config.OwnCloudSQLDriver{
 				Root:                  filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"),

services/storage-users/pkg/revaconfig/config.go

@@ -1,3 +1,4 @@
+// Package revaconfig contains the config for the reva service
 package revaconfig
 
 import (
@@ -10,11 +11,12 @@ import (
 func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} {
 	rcfg := map[string]interface{}{
 		"core": map[string]interface{}{
-			"tracing_enabled":      cfg.Tracing.Enabled,
-			"tracing_exporter":     cfg.Tracing.Type,
-			"tracing_endpoint":     cfg.Tracing.Endpoint,
-			"tracing_collector":    cfg.Tracing.Collector,
-			"tracing_service_name": cfg.Service.Name,
+			"tracing_enabled":           cfg.Tracing.Enabled,
+			"tracing_exporter":          cfg.Tracing.Type,
+			"tracing_endpoint":          cfg.Tracing.Endpoint,
+			"tracing_collector":         cfg.Tracing.Collector,
+			"tracing_service_name":      cfg.Service.Name,
+			"graceful_shutdown_timeout": cfg.GracefulShutdownTimeout,
 		},
 		"shared": map[string]interface{}{
 			"jwt_secret":                cfg.TokenManager.JWTSecret,

vendor/modules.txt

@@ -352,7 +352,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
 github.com/cs3org/go-cs3apis/cs3/types/v1beta1
-# github.com/cs3org/reva/v2 v2.15.0
+# github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9
 ## explicit; go 1.20
 github.com/cs3org/reva/v2/cmd/revad/internal/grace
 github.com/cs3org/reva/v2/cmd/revad/runtime