Skip to content

Commit

Permalink
REST server: http log filtering and request read timeout (#102)
Browse files Browse the repository at this point in the history 
* REST server: http log filtering and request read timeout

1) Standard http library writes error and warning messages like "TLS
handshake error" to staderr, which gets forwarded to syslog.  It can
flood the syslog during stress tests.

Added an error log writer to the REST http server to capture all such
messages and filter out trivial messages. Only accept failures and
handler panics are treated as non-trivial. All other messages are
logged only if glog verbose level 2 is enabled.

2) Added a configurable 'read timeout' to REST server which is the
deadline for reading a full http request (TLS+header+body) after a new
connection is accepted. Connection is dropped if it was left idle for
more than this duration or client takes longer to push the request.
Default vale is 15s; can be changed via '-readtimeout T' command line
flag. Value 0 disables this timeout.

* Change startup settings log to a verbose log
  • Loading branch information
@sachinholla
sachinholla authored May 31, 2023
1 parent 1303b05 commit c36f6e1
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 4 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/gorilla/mux v1.7.4
github.com/leodido/go-urn v1.2.0 // indirect
github.com/pkg/profile v1.4.0
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
gopkg.in/go-playground/validator.v9 v9.31.0
)

Expand Down
13 changes: 13 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@ github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1
github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA=
github.com/antchfx/jsonquery v1.1.0 h1:ZeqeHheI8WsEN5blUqZXZ30w2jrbFvlQIq5B7X7Z86E=
github.com/antchfx/jsonquery v1.1.0/go.mod h1:h7950pvPrUZzJIflNqsELgDQovTpPNa0rAHf8NwjegY=
github.com/antchfx/jsonquery v1.1.4 h1:+OlFO3QS9wjU0MKx9MgHm5f6o6hdd4e9mUTp0wTjxlM=
github.com/antchfx/jsonquery v1.1.4/go.mod h1:cHs8r6Bymd8j6HI6Ej1IJbjahKvLBcIEh54dfmo+E9A=
github.com/antchfx/xmlquery v1.3.1 h1:nIKWdtnhrXtj0/IRUAAw2I7TfpHUa3zMnHvNmPXFg+w=
github.com/antchfx/xmlquery v1.3.1/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
github.com/antchfx/xpath v1.1.2 h1:YziPrtM0gEJBnhdUGxYcIVYXZ8FXbtbovxOi+UW/yWQ=
github.com/antchfx/xpath v1.1.2/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antchfx/xpath v1.1.7/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg=
github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
Expand All @@ -26,6 +31,8 @@ github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be
github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.0.0-beta.3.0.20190824101152-d19aba07b476 h1:WNSiFp8Ww4ZP7XUzW56zDYv5roKQ4VfsdHCLoh8oDj4=
github.com/go-redis/redis/v7 v7.0.0-beta.3.0.20190824101152-d19aba07b476/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
Expand Down Expand Up @@ -85,6 +92,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
Expand All @@ -98,6 +107,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2eP
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -107,6 +118,8 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
Expand Down
47 changes: 44 additions & 3 deletions rest/main/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,19 @@
package main

import (
"bytes"
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"

"github.com/Azure/sonic-mgmt-framework/build/rest_server/dist/swagger"
"github.com/Azure/sonic-mgmt-framework/rest/server"
Expand All @@ -44,6 +47,10 @@ var (
keyFile string // Server private key file path
caFile string // Client CA certificate file path
clientAuth string // Client auth mode

// readTimeout is the deadline for receiving a full request (TLS+header+body)
// once the connection is made. Value 0 indicates no timeout.
readTimeout time.Duration = 15 * time.Second
)

func init() {
Expand All @@ -54,6 +61,7 @@ func init() {
flag.StringVar(&keyFile, "key", "", "Server private key file path")
flag.StringVar(&caFile, "cacert", "", "CA certificate for client certificate validation")
flag.StringVar(&clientAuth, "client_auth", "none", "Client auth mode - none|cert|user")
flag.DurationVar(&readTimeout, "readtimeout", readTimeout, "Maximum duration for reading entire request")
flag.Parse()
}

Expand Down Expand Up @@ -103,9 +111,16 @@ func main() {

// Prepare HTTPS server
restServer := &http.Server{
Addr: address,
Handler: router,
TLSConfig: &tlsConfig,
Addr: address,
Handler: router,
TLSConfig: &tlsConfig,
ReadTimeout: readTimeout,
ErrorLog: serverLog,
}

if glog.V(1) {
glog.Infof("Read timeout = %v", readTimeout)
glog.Infof("Authentication modes = %v", clientAuth)
}

glog.Infof("Server started on %v", address)
Expand Down Expand Up @@ -218,3 +233,29 @@ func findAManagementIP() string {
glog.Warning("Could not find a management address!!")
return ""
}

// errorMsgPrefixes identifies the important error messages logged by
// the standard http library that should not be missed
var errorMsgPrefixes = [][]byte{
[]byte("http: Accept error"),
[]byte("http: panic"),
[]byte("http2: panic"),
}

// serverLog forwards the http library's logs to glog
var serverLog = log.New(logWriter{}, "", 0)

type logWriter struct{}

func (logWriter) Write(p []byte) (int, error) {
for _, pfx := range errorMsgPrefixes {
if bytes.HasPrefix(p, pfx) {
glog.Warning(string(p))
return len(p), nil
}
}
if glog.V(2) {
glog.Info(string(p))
}
return len(p), nil
}

0 comments on commit c36f6e1

Please sign in to comment.