From 94ae72dc1b8dc4f5c08cc033d074bf715829a8e0 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 11:15:51 +0200 Subject: [PATCH 1/9] Start grpc server and register services when starting in standalone mode --- server/start.go | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/server/start.go b/server/start.go index 061b736cf6a..3b4f1da2bd1 100644 --- a/server/start.go +++ b/server/start.go @@ -142,7 +142,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. if !withTM { serverCtx.Logger.Info("starting ABCI without Tendermint") return wrapCPUProfile(serverCtx, func() error { - return startStandAlone(serverCtx, appCreator) + return startStandAlone(serverCtx, clientCtx, appCreator) }) } @@ -206,7 +206,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. return cmd } -func startStandAlone(ctx *Context, appCreator types.AppCreator) error { +func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.AppCreator) error { addr := ctx.Viper.GetString(flagAddress) transport := ctx.Viper.GetString(flagTransport) home := ctx.Viper.GetString(flags.FlagHome) @@ -229,11 +229,45 @@ func startStandAlone(ctx *Context, appCreator types.AppCreator) error { return err } + // Add the tx service to the gRPC router. We only need to register this + // service if gRPC is enabled + if config.GRPC.Enable { + // use the clientCtx provided to start command + app.RegisterTxService(clientCtx) + app.RegisterTendermintService(clientCtx) + app.RegisterNodeService(clientCtx) + } + _, err = startTelemetry(config) if err != nil { return err } + var ( + grpcSrv *grpc.Server + grpcWebSrv *http.Server + ) + + if config.GRPC.Enable { + grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) + if err != nil { + return err + } + defer grpcSrv.Stop() + if config.GRPCWeb.Enable { + grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config) + if err != nil { + ctx.Logger.Error("failed to start grpc-web http server: ", err) + return err + } + defer func() { + if err := grpcWebSrv.Close(); err != nil { + ctx.Logger.Error("failed to close grpc-web http server: ", err) + } + }() + } + } + svr, err := server.NewServer(addr, transport, app) if err != nil { return fmt.Errorf("error creating listener: %v", err) From 07a7a4e3a1c1bb66065d15b0d331bdb923eb4641 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 13:40:46 +0200 Subject: [PATCH 2/9] Create new client from context in standalone mode --- server/start.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/start.go b/server/start.go index 3b4f1da2bd1..b00a9e7a910 100644 --- a/server/start.go +++ b/server/start.go @@ -35,6 +35,8 @@ import ( "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/mempool" + + rpchttp "github.com/cometbft/cometbft/rpc/client/http" ) const ( @@ -232,6 +234,14 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap // Add the tx service to the gRPC router. We only need to register this // service if gRPC is enabled if config.GRPC.Enable { + // create tendermint client + rpcclient, err := rpchttp.New(ctx.Config.RPC.ListenAddress, "/websocket") + if err != nil { + return err + } + // re-assign for making the client available below + // do not use := to avoid shadowing clientCtx + clientCtx = clientCtx.WithClient(rpcclient) // use the clientCtx provided to start command app.RegisterTxService(clientCtx) app.RegisterTendermintService(clientCtx) From 3f90a1de79d538f3acfc53dc5045f6c321183b0f Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 13:42:27 +0200 Subject: [PATCH 3/9] Add comment to explain listen address --- server/start.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/start.go b/server/start.go index b00a9e7a910..e1da6e67dca 100644 --- a/server/start.go +++ b/server/start.go @@ -235,6 +235,7 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap // service if gRPC is enabled if config.GRPC.Enable { // create tendermint client + // assumes the rpc listen address is where tendermint has its rpc server rpcclient, err := rpchttp.New(ctx.Config.RPC.ListenAddress, "/websocket") if err != nil { return err From 2a72ca086539b5b80d7bd7295b00a8aeecc0d442 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 16:50:57 +0200 Subject: [PATCH 4/9] Add paragraph about standalone CometBFT to README --- docs/docs/run-node/01-run-node.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/docs/run-node/01-run-node.md b/docs/docs/run-node/01-run-node.md index cce2bed5fa6..fdda4411c86 100644 --- a/docs/docs/run-node/01-run-node.md +++ b/docs/docs/run-node/01-run-node.md @@ -136,6 +136,13 @@ The previous command allow you to run a single node. This is enough for the next The naive way would be to run the same commands again in separate terminal windows. This is possible, however in the Cosmos SDK, we leverage the power of [Docker Compose](https://docs.docker.com/compose/) to run a localnet. If you need inspiration on how to set up your own localnet with Docker Compose, you can have a look at the Cosmos SDK's [`docker-compose.yml`](https://github.com/cosmos/cosmos-sdk/blob/v0.47.0-rc1/docker-compose.yml). +### Standalone App/CometBFT + +By default, the Cosmos SDK runs CometBFT in-process with the application +If you want to run the application and CometBFT in separate processes, +start the application with the `--with-comet=false` flag +and set `rpc.laddr` in `config.toml` to the CometBFT node's RPC address. + ## Logging Logging provides a way to see what is going on with a node. By default the info level is set. This is a global level and all info logs will be outputted to the terminal. If you would like to filter specific logs to the terminal instead of all, then setting `module:log_level` is how this can work. From 18bdec5ec32f81c57a797349cac0e763a5e80e14 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 16 Oct 2023 09:10:55 +0200 Subject: [PATCH 5/9] Add changelog entry for standalone gRPC --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02203d95b8b..228e58586c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,10 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Features + +* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC server in standalone mode + ## [v0.47.5](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.47.5) - 2023-09-01 ### Features From b56e95ea93c4e2cbd4e8afab9da7866293335394 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 16 Oct 2023 10:15:11 +0200 Subject: [PATCH 6/9] Change exit outside of deferred func to error return --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index e1da6e67dca..9633c9b25c1 100644 --- a/server/start.go +++ b/server/start.go @@ -289,7 +289,7 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap err = svr.Start() if err != nil { fmt.Println(err.Error()) - os.Exit(1) + return err } defer func() { From 4ef3398d775ac695d587bdd3da87e5b1bba44349 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 16 Oct 2023 15:02:41 +0200 Subject: [PATCH 7/9] Start the API server in standalone mode --- server/start.go | 160 ++++++++++++++++++++++++++---------------------- 1 file changed, 88 insertions(+), 72 deletions(-) diff --git a/server/start.go b/server/start.go index 9633c9b25c1..cee95b65d43 100644 --- a/server/start.go +++ b/server/start.go @@ -249,7 +249,16 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap app.RegisterNodeService(clientCtx) } - _, err = startTelemetry(config) + metrics, err := startTelemetry(config) + if err != nil { + return err + } + + cfg := ctx.Config + + genDocProvider := node.DefaultGenesisDocProviderFunc(cfg) + + apiSrv, err := startAPIserver(config, genDocProvider, clientCtx, home, ctx, app, metrics) if err != nil { return err } @@ -293,14 +302,12 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap } defer func() { - if err = svr.Stop(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } + _ = svr.Stop() + + _ = app.Close() - if err = app.Close(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) + if apiSrv != nil { + _ = apiSrv.Close() } }() @@ -399,70 +406,9 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - var apiSrv *api.Server - if config.API.Enable { - genDoc, err := genDocProvider() - if err != nil { - return err - } - - clientCtx := clientCtx.WithHomeDir(home).WithChainID(genDoc.ChainID) - - if config.GRPC.Enable { - _, port, err := net.SplitHostPort(config.GRPC.Address) - if err != nil { - return err - } - - maxSendMsgSize := config.GRPC.MaxSendMsgSize - if maxSendMsgSize == 0 { - maxSendMsgSize = serverconfig.DefaultGRPCMaxSendMsgSize - } - - maxRecvMsgSize := config.GRPC.MaxRecvMsgSize - if maxRecvMsgSize == 0 { - maxRecvMsgSize = serverconfig.DefaultGRPCMaxRecvMsgSize - } - - grpcAddress := fmt.Sprintf("127.0.0.1:%s", port) - - // If grpc is enabled, configure grpc client for grpc gateway. - grpcClient, err := grpc.Dial( - grpcAddress, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultCallOptions( - grpc.ForceCodec(codec.NewProtoCodec(clientCtx.InterfaceRegistry).GRPCCodec()), - grpc.MaxCallRecvMsgSize(maxRecvMsgSize), - grpc.MaxCallSendMsgSize(maxSendMsgSize), - ), - ) - if err != nil { - return err - } - - clientCtx = clientCtx.WithGRPCClient(grpcClient) - ctx.Logger.Debug("grpc client assigned to client context", "target", grpcAddress) - } - - apiSrv = api.New(clientCtx, ctx.Logger.With("module", "api-server")) - app.RegisterAPIRoutes(apiSrv, config.API) - if config.Telemetry.Enabled { - apiSrv.SetTelemetry(metrics) - } - errCh := make(chan error) - - go func() { - if err := apiSrv.Start(config); err != nil { - errCh <- err - } - }() - - select { - case err := <-errCh: - return err - - case <-time.After(types.ServerStartTime): // assume server started successfully - } + apiSrv, err := startAPIserver(config, genDocProvider, clientCtx, home, ctx, app, metrics) + if err != nil { + return err } var ( @@ -568,6 +514,76 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return WaitForQuitSignals() } +func startAPIserver(config serverconfig.Config, genDocProvider node.GenesisDocProvider, clientCtx client.Context, home string, ctx *Context, app types.Application, metrics *telemetry.Metrics) (*api.Server, error) { + // If grpc is enabled, configure grpc client for grpc gateway. + // assume server started successfully + var apiSrv *api.Server + if config.API.Enable { + genDoc, err := genDocProvider() + if err != nil { + return nil, err + } + + clientCtx := clientCtx.WithHomeDir(home).WithChainID(genDoc.ChainID) + + if config.GRPC.Enable { + _, port, err := net.SplitHostPort(config.GRPC.Address) + if err != nil { + return nil, err + } + + maxSendMsgSize := config.GRPC.MaxSendMsgSize + if maxSendMsgSize == 0 { + maxSendMsgSize = serverconfig.DefaultGRPCMaxSendMsgSize + } + + maxRecvMsgSize := config.GRPC.MaxRecvMsgSize + if maxRecvMsgSize == 0 { + maxRecvMsgSize = serverconfig.DefaultGRPCMaxRecvMsgSize + } + + grpcAddress := fmt.Sprintf("127.0.0.1:%s", port) + + grpcClient, err := grpc.Dial( + grpcAddress, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.ForceCodec(codec.NewProtoCodec(clientCtx.InterfaceRegistry).GRPCCodec()), + grpc.MaxCallRecvMsgSize(maxRecvMsgSize), + grpc.MaxCallSendMsgSize(maxSendMsgSize), + ), + ) + if err != nil { + return nil, err + } + + clientCtx = clientCtx.WithGRPCClient(grpcClient) + ctx.Logger.Debug("grpc client assigned to client context", "target", grpcAddress) + } + + apiSrv = api.New(clientCtx, ctx.Logger.With("module", "api-server")) + app.RegisterAPIRoutes(apiSrv, config.API) + if config.Telemetry.Enabled { + apiSrv.SetTelemetry(metrics) + } + errCh := make(chan error) + + go func() { + if err := apiSrv.Start(config); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + return nil, err + + case <-time.After(types.ServerStartTime): + } + } + return apiSrv, nil +} + func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { if !cfg.Telemetry.Enabled { return nil, nil From 8b49a986a77ff1e891624d8902fc68c9475a4ec5 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 18 Oct 2023 12:40:26 +0200 Subject: [PATCH 8/9] Also register services if API server is started --- server/start.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/start.go b/server/start.go index cee95b65d43..31deb30745d 100644 --- a/server/start.go +++ b/server/start.go @@ -232,8 +232,9 @@ func startStandAlone(ctx *Context, clientCtx client.Context, appCreator types.Ap } // Add the tx service to the gRPC router. We only need to register this - // service if gRPC is enabled - if config.GRPC.Enable { + // service if API or gRPC is enabled, and avoid doing so in the general + // case, because it spawns a new local CometBFT RPC client. + if config.API.Enable || config.GRPC.Enable { // create tendermint client // assumes the rpc listen address is where tendermint has its rpc server rpcclient, err := rpchttp.New(ctx.Config.RPC.ListenAddress, "/websocket") From f4cedc19881d9b1f7cfbe6acab93fda64d7617c1 Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 19 Oct 2023 10:22:20 +0200 Subject: [PATCH 9/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72c82ebd0b1..b496daa0309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features -* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC server in standalone mode +* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC & API server in standalone mode ### Improvements