Skip to content
This repository has been archived by the owner on Jun 20, 2024. It is now read-only.

Bifrost panic in staging #136

Closed
aarshkshah1992 opened this issue May 29, 2023 · 2 comments · Fixed by ipfs/boxo#319 or #137
Closed

Bifrost panic in staging #136

aarshkshah1992 opened this issue May 29, 2023 · 2 comments · Fixed by ipfs/boxo#319 or #137

Comments

@aarshkshah1992
Copy link
Collaborator

aarshkshah1992 commented May 29, 2023 •
edited
Loading

Noticed this Bifrost panic in staging.

Commit hash b4ddd33 which is just some Caboose config changes on the top of the current Bifrost main branch

Note: This does NOT happen everytime on startup and can't be reproduced reliably.

2023-05-29T17:47:51.537Z	ERROR	core/server	gateway/handler.go:330	A panic occurred in the gateway handler!
2023-05-29T17:47:51.537Z	ERROR	core/server	gateway/handler.go:331	runtime error: invalid memory address or nil pointer dereference
goroutine 6513396 [running]:
runtime/debug.Stack()
	/usr/local/go/src/runtime/debug/stack.go:24 +0x65
runtime/debug.PrintStack()
	/usr/local/go/src/runtime/debug/stack.go:16 +0x19
github.com/ipfs/boxo/gateway.panicHandler({0x1414b20, 0xc102d67920})
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:332 +0xfc
panic({0xec49e0, 0x1b17210})
	/usr/local/go/src/runtime/panic.go:884 +0x213
github.com/ipfs/boxo/gateway.(*handler).serveDefaults(0xc0009af680, {0x1414f10, 0xc163f7a480}, {0x1414b20, 0xc102d67920}, 0xc145348aa8, {{0x1415140?, 0xc1041651b0?}}, {{0x1415140, 0xc1041651b0}}, ...)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler_defaults.go:86 +0x7cb
github.com/ipfs/boxo/gateway.(*handler).getOrHeadHandler(0xc0009af680, {0x1414b20, 0xc102d67920}, 0xc015ef8cc8)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:296 +0xd74
github.com/ipfs/boxo/gateway.(*handler).ServeHTTP(0xc0009af680, {0x1414b20, 0xc102d67920}, 0xc160ecf900)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:168 +0x234
net/http.(*ServeMux).ServeHTTP(0xe69780?, {0x1414b20, 0xc102d67920}, 0xc160ecf900)
	/usr/local/go/src/net/http/server.go:2500 +0x149
main.withConnect.func1({0x1414b20?, 0xc102d67920?}, 0xc163f7a360?)
	/go/src/github.com/ipfs/bifrost-gateway/handlers.go:58 +0x73
net/http.HandlerFunc.ServeHTTP(0xc160ecf600?, {0x1414b20?, 0xc102d67920?}, 0xc17742bc04?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/ipfs/boxo/gateway.WithHostname.func1({0x1414b20, 0xc102d67920}, 0xc160ecf600)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/hostname.go:222 +0xa18
net/http.HandlerFunc.ServeHTTP(0x0?, {0x1414b20?, 0xc102d67920?}, 0x0?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/mitchellh/go-server-timing.Middleware.func1({0x7f3934c51328, 0xc085b4ce60}, 0xc160ecf500)
	/go/pkg/mod/github.com/mitchellh/go-server-timing@v1.0.1/middleware.go:74 +0x32b
net/http.HandlerFunc.ServeHTTP(0x1414b20?, {0x7f3934c51328?, 0xc085b4ce60?}, 0x0?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x1414b20?, 0xc102d678c0?}, 0xc160ecf500)
	/go/pkg/mod/github.com/prometheus/client_golang@v1.14.0/prometheus/promhttp/instrument_server.go:288 +0xc5
net/http.HandlerFunc.ServeHTTP(0xc0f7af3450?, {0x1414b20?, 0xc102d678c0?}, 0xc0000eb520?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
main.withRequestLogger.func1({0x1414b20, 0xc102d678c0}, 0xc160ecf500)
	/go/src/github.com/ipfs/bifrost-gateway/handlers.go:66 +0x171
net/http.HandlerFunc.ServeHTTP(0x1414f10?, {0x1414b20?, 0xc102d678c0?}, 0x1406450?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Handler).ServeHTTP(0xc000254480, {0x1413a40?, 0xc117c047e0}, 0xc160ecf400)
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.40.0/handler.go:213 +0x1231
net/http.serverHandler.ServeHTTP({0xc163f7a1b0?}, {0x1413a40, 0xc117c047e0}, 0xc160ecf400)
	/usr/local/go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc15da7cf30, {0x1414f10, 0xc0001d6180})
	/usr/local/go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3089 +0x5ed
2023/05/29 17:48:33 http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192
2023/05/29 17:48:36 http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192
2023/05/29 17:48:36 http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192
2023-05-29T17:48:51.576Z	ERROR	core/server	gateway/handler.go:330	A panic occurred in the gateway handler!
2023-05-29T17:48:51.576Z	ERROR	core/server	gateway/handler.go:331	runtime error: invalid memory address or nil pointer dereference
goroutine 6817028 [running]:
runtime/debug.Stack()
	/usr/local/go/src/runtime/debug/stack.go:24 +0x65
runtime/debug.PrintStack()
	/usr/local/go/src/runtime/debug/stack.go:16 +0x19
github.com/ipfs/boxo/gateway.panicHandler({0x1414b20, 0xc0f4198ea0})
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:332 +0xfc
panic({0xec49e0, 0x1b17210})
	/usr/local/go/src/runtime/panic.go:884 +0x213
github.com/ipfs/boxo/gateway.(*handler).serveDefaults(0xc0009af680, {0x1414f10, 0xc1573b21e0}, {0x1414b20, 0xc0f4198ea0}, 0xc012220aa8, {{0x1415140?, 0xc1773c7740?}}, {{0x1415140, 0xc1773c7740}}, ...)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler_defaults.go:86 +0x7cb
github.com/ipfs/boxo/gateway.(*handler).getOrHeadHandler(0xc0009af680, {0x1414b20, 0xc0f4198ea0}, 0xc01a38acc8)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:296 +0xd74
github.com/ipfs/boxo/gateway.(*handler).ServeHTTP(0xc0009af680, {0x1414b20, 0xc0f4198ea0}, 0xc18a48df00)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/handler.go:168 +0x234
net/http.(*ServeMux).ServeHTTP(0xe69780?, {0x1414b20, 0xc0f4198ea0}, 0xc18a48df00)
	/usr/local/go/src/net/http/server.go:2500 +0x149
main.withConnect.func1({0x1414b20?, 0xc0f4198ea0?}, 0xc1573b2000?)
	/go/src/github.com/ipfs/bifrost-gateway/handlers.go:58 +0x73
net/http.HandlerFunc.ServeHTTP(0xc18a48de00?, {0x1414b20?, 0xc0f4198ea0?}, 0xc187b1a154?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/ipfs/boxo/gateway.WithHostname.func1({0x1414b20, 0xc0f4198ea0}, 0xc18a48de00)
	/go/pkg/mod/github.com/ipfs/boxo@v0.8.2-0.20230522115955-e4682237d5e3/gateway/hostname.go:222 +0xa18
net/http.HandlerFunc.ServeHTTP(0x0?, {0x1414b20?, 0xc0f4198ea0?}, 0x0?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/mitchellh/go-server-timing.Middleware.func1({0x7f3934c51328, 0xc0615fdb80}, 0xc18a48dd00)
	/go/pkg/mod/github.com/mitchellh/go-server-timing@v1.0.1/middleware.go:74 +0x32b
net/http.HandlerFunc.ServeHTTP(0x1414b20?, {0x7f3934c51328?, 0xc0615fdb80?}, 0x0?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x1414b20?, 0xc0f4198e40?}, 0xc18a48dd00)
	/go/pkg/mod/github.com/prometheus/client_golang@v1.14.0/prometheus/promhttp/instrument_server.go:288 +0xc5
net/http.HandlerFunc.ServeHTTP(0xc16fd95450?, {0x1414b20?, 0xc0f4198e40?}, 0xc0000eb520?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
main.withRequestLogger.func1({0x1414b20, 0xc0f4198e40}, 0xc18a48dd00)
	/go/src/github.com/ipfs/bifrost-gateway/handlers.go:66 +0x171
net/http.HandlerFunc.ServeHTTP(0x1414f10?, {0x1414b20?, 0xc0f4198e40?}, 0x1406450?)
	/usr/local/go/src/net/http/server.go:2122 +0x2f
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Handler).ServeHTTP(0xc000254480, {0x1413a40?, 0xc100f72540}, 0xc18a48da00)
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.40.0/handler.go:213 +0x1231
net/http.serverHandler.ServeHTTP({0xc0fe7d7b00?}, {0x1413a40, 0xc100f72540}, 0xc18a48da00)
	/usr/local/go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc14d0de2d0, {0x1414f10, 0xc0001d6180})
	/usr/local/go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3089 +0x5ed
@willscott
Copy link
Collaborator

looks like it's

https://github.com/ipfs/boxo/blob/main/gateway/handler_defaults.go#L86

is it that in the error handler above we may have an error but then not return?
https://github.com/ipfs/boxo/blob/main/gateway/handler_defaults.go#L77-L78

@lidel lidel mentioned this issue May 29, 2023
Merged
lidel added a commit that referenced this issue May 29, 2023
@lidel
fix: panic in get
74f3eb0 
Includes ipfs/boxo#319
Closes #136
@lidel lidel mentioned this issue May 29, 2023
Merged
@BigLep
Copy link

BigLep commented Jun 3, 2023

Thanks for debugging and handling @aarshkshah1992 and @willscott .

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

fix(gw): missing return in error case ipfs/boxo
fix: panic in get ipfs-inactive/bifrost-gateway
3 participants
@BigLep @willscott @aarshkshah1992