runtime: "morestack on g0" in debugcall on android-amd64

[prattmic]

Seen once on gomote with all.bash:

fatal: morestack on g0
trap in unknown function runtime.abort
SIGTRAP: trace trap
PC=0x7e7838bba342 m=11 sigcode=128

goroutine 0 [idle]:
runtime.abort()
        /workdir/go/src/runtime/asm_amd64.s:1059 +0x2 fp=0x7e780fafd410 sp=0x7e780fafd408 pc=0x7e7838bba342
runtime.morestack()
        /workdir/go/src/runtime/asm_amd64.s:538 +0x22 fp=0x7e780fafd418 sp=0x7e780fafd410 pc=0x7e7838bb8322

goroutine 23000 [running, locked to thread]:
runtime.systemstack_switch()
        /workdir/go/src/runtime/asm_amd64.s:459 fp=0xc0024ba620 sp=0xc0024ba618 pc=0x7e7838bb8260
runtime.debugCallCheck(0x7e7838e0bce5)
        /workdir/go/src/runtime/debugcall.go:42 +0x85 fp=0xc0024ba660 sp=0xc0024ba620 pc=0x7e7838b4c765
runtime.debugCallCheck(0x7e7838e0bce5)
        <autogenerated>:1 +0x27 fp=0xc0024ba678 sp=0xc0024ba660 pc=0x7e7838bbca07
runtime.debugCallV2()
        /workdir/go/src/runtime/asm_amd64.s:1833 +0x89 fp=0xc0024ba720 sp=0xc0024ba678 pc=0x7e7838bba789
runtime_test.debugCallWorker2(0xc000c3bb00?, 0xc0000343d8?)
        /workdir/go/src/runtime/debug_test.go:94 +0x5 fp=0xc0024ba728 sp=0xc0024ba720 pc=0x7e7838e0bce5
runtime_test.debugCallWorker(0xc000ef84e0?, 0x3e7?, 0xc000034380?)
        /workdir/go/src/runtime/debug_test.go:79 +0x8b fp=0xc0024ba7b8 sp=0xc0024ba728 pc=0x7e7838e0bc0b
runtime_test.startDebugCallWorker.func2()
        /workdir/go/src/runtime/debug_test.go:59 +0x2e fp=0xc0024ba7e0 sp=0xc0024ba7b8 pc=0x7e7838e0bb4e
runtime.goexit()
        /workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0024ba7e8 sp=0xc0024ba7e0 pc=0x7e7838bba501
created by runtime_test.startDebugCallWorker
        /workdir/go/src/runtime/debug_test.go:59 +0xee

This looks like morestack right on entry to this closure, which is odd because we should have plenty of stack.

cc @golang/runtime @mknyszek

[cherrymui]

Yeah, that call is made on the system stack. From the stack trace it seems it successfully switched to the g0 stack. Maybe the stack bounds of g0 is wrong?

[ManouchehrRasoulli]

same issue, with godror integration with instant client ! service build successfully, and every things happen in runtime for giving context, following service work correctly localy on k8s after few pod restart service became online.

fatal: morestack on g0
SIGTRAP: trace trap
PC=0x4730e2 m=11 sigcode=128
signal arrived during cgo execution
goroutine 1 [syscall, locked to thread]:
runtime.cgocall(0xda9b10, 0xc000615130)
	/usr/local/go/src/runtime/cgocall.go:156 +0x5c fp=0xc000615108 sp=0xc0006150d0 pc=0x411a5c
github.com/godror/godror._Cfunc_dpiPool_create(0x1e206e0, 0x1e207a0, 0x4, 0x1e207c0, 0x13, 0x1e20800, 0x22, 0xc0002f8500, 0xc000320b40, 0xc000010130)
	_cgo_gotypes.go:4421 +0x4c fp=0xc000615130 sp=0xc000615108 pc=0x6f80ec
github.com/godror/godror.(*drv).createPool.func5.1(0x7f0e72421108, 0x10, 0x10, 0x100000e991e0a68, 0x7f0e991e93b0, 0x8, 0x8, 0x7f0e991e0a68)
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:773 +0x165 fp=0xc0006151e8 sp=0xc000615130 pc=0x70d945
github.com/godror/godror.(*drv).createPool.func5()
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:773 +0x39 fp=0xc000615238 sp=0xc0006151e8 pc=0x70d7b9
github.com/godror/godror.(*drv).checkExecNoLOT(0x41b707, 0x8)
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:254 +0x22 fp=0xc000615258 sp=0xc000615238 pc=0x708e02
github.com/godror/godror.(*drv).checkExec(0x1e20800, 0x22)
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:248 +0x34 fp=0xc000615288 sp=0xc000615258 pc=0x708d54
github.com/godror/godror.(*drv).createPool(_, {{{0xc00029e99c, 0x4}, {0xc00031a2d0, 0x22}, {{0xc000040bd0, 0x13}}, {0x0, 0x0}, {0x0, ...}, ...}, ...})
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:765 +0x772 fp=0xc0006154b0 sp=0xc000615288 pc=0x70d3b2
github.com/godror/godror.(*drv).getPool(_, {{{0xc00029e99c, 0x4}, {0xc00031a2d0, 0x22}, {{0xc000040bd0, 0x13}}, {0x0, 0x0}, {0x0, ...}, ...}, ...})
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:656 +0x579 fp=0xc000615760 sp=0xc0006154b0 pc=0x70c8f9
github.com/godror/godror.(*drv).createConnFromParams(_, {{{0xc00029e99c, 0x4}, {0xc00031a2d0, 0x22}, {{0xc000040bd0, 0x13}}, {0x0, 0x0}, {0x0, ...}, ...}, ...})
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:591 +0x198 fp=0xc000615c10 sp=0xc000615760 pc=0x70c138
github.com/godror/godror.connector.Connect({0x1987d60, {{{0xc00029e99c, 0x4}, {0xc00031a2d0, 0x22}, {{...}}, {0x0, 0x0}, {0x0, 0x0}, ...}, ...}}, ...)
	/go/pkg/mod/github.com/godror/godror@v0.34.0/drv.go:1113 +0x7b8 fp=0xc0006163c8 sp=0xc000615c10 pc=0x710238
github.com/godror/godror.(*connector).Connect(0xc0003dc160, {0x122cb88, 0xc000046050})
	<autogenerated>:1 +0x9e fp=0xc0006166a8 sp=0xc0006163c8 pc=0x7520be
database/sql.(*DB).conn(0xc0003ae820, {0x122cb88, 0xc000046050}, 0x1)
	/usr/local/go/src/database/sql/sql.go:1364 +0x7ac fp=0xc000616810 sp=0xc0006166a8 pc=0x500d4c
database/sql.(*DB).PingContext(0xfa76f6, {0x122cb88, 0xc000046050})
	/usr/local/go/src/database/sql/sql.go:853 +0x7a fp=0xc000616898 sp=0xc000616810 pc=0x4fe8da
database/sql.(*DB).Ping(...)
	/usr/local/go/src/database/sql/sql.go:874
main.newDB(0xc0007a1500, {0x1234218, 0xc000558568})
	/builds/ap/platform/mobapp-auth/cmd/server.go:313 +0x4b8 fp=0xc000616ac8 sp=0xc000616898 pc=0xd80738
main.NewServer()
	/builds/ap/platform/mobapp-auth/cmd/server.go:141 +0x376 fp=0xc000617f60 sp=0xc000616ac8 pc=0xd7e816
main.main()
	/builds/ap/platform/mobapp-auth/cmd/server.go:47 +0x19 fp=0xc000617f80 sp=0xc000617f60 pc=0xd7df19
runtime.main()
	/usr/local/go/src/runtime/proc.go:255 +0x227 fp=0xc000617fe0 sp=0xc000617f80 pc=0x4453c7
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000617fe8 sp=0xc000617fe0 pc=0x473261
goroutine 30 [chan receive]:
github.com/getsentry/sentry-go.(*HTTPTransport).worker(0xc0001f6b00)
	/go/pkg/mod/github.com/getsentry/sentry-go@v0.9.0/transport.go:365 +0x126
created by github.com/getsentry/sentry-go.(*HTTPTransport).Configure.func1
	/go/pkg/mod/github.com/getsentry/sentry-go@v0.9.0/transport.go:236 +0x5f
goroutine 83 [select]:
database/sql.(*DB).connectionCleaner(0xc00085e1a0, 0xc000050001)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 48 [select]:
go.mongodb.org/mongo-driver/internal.(*CancellationListener).Listen(0xc000116810, {0x122cb50, 0xc000850500}, 0xc0003b4420)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/internal/cancellation_listener.go:30 +0x7e
created by go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*connection).read
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/connection.go:405 +0x17b
goroutine 31 [select]:
go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*rttMonitor).start(0xc0000acd20)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/rtt_monitor.go:87 +0x13e
created by go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*rttMonitor).connect
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/rtt_monitor.go:50 +0x6f
goroutine 32 [IO wait]:
internal/poll.runtime_pollWait(0x7f0e72466620, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0001f6c00, 0xc00029e848, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0001f6c00, {0xc00029e848, 0x4, 0x4})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0001f6c00, {0xc00029e848, 0x41b3b4, 0x7f0e72466700})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000100f8, {0xc00029e848, 0x7f0e991e0a68, 0x30})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x1214060, 0xc0000100f8}, {0xc00029e848, 0x4, 0x4}, 0x4)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*connection).read(0xc000264780, {0x122cb50, 0xc000850500}, {0xc00010c400, 0x445796, 0x100})
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/connection.go:424 +0x225
go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*connection).readWireMessage(0xc000264780, {0x122cb50, 0xc000850500}, {0xc00010c400, 0x0, 0x100})
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/connection.go:385 +0x207
go.mongodb.org/mongo-driver/x/mongo/driver/topology.initConnection.ReadWireMessage({0xd2817d}, {0x122cb50, 0xc000850500}, {0xc00010c400, 0x0, 0x0})
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/connection.go:548 +0x28
go.mongodb.org/mongo-driver/x/mongo/driver.Operation.readWireMessage({0xc0003b43d0, {0xfa61f7, 0x5}, {0x1223f68, 0xc0003b43c0}, 0xc0003b43e0, {0x0, 0x0}, 0x0, 0x0, ...}, ...)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/operation.go:603 +0x7e
go.mongodb.org/mongo-driver/x/mongo/driver.Operation.roundTrip({0xc0003b43d0, {0xfa61f7, 0x5}, {0x1223f68, 0xc0003b43c0}, 0xc0003b43e0, {0x0, 0x0}, 0x0, 0x0, ...}, ...)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/operation.go:597 +0x330
go.mongodb.org/mongo-driver/x/mongo/driver.Operation.Execute({0xc0003b43d0, {0xfa61f7, 0x5}, {0x1223f68, 0xc0003b43c0}, 0xc0003b43e0, {0x0, 0x0}, 0x0, 0x0, ...}, ...)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/operation.go:368 +0x11fc
go.mongodb.org/mongo-driver/x/mongo/driver/operation.(*IsMaster).Execute(0xc0003be360, {0x122cb50, 0xc000850500})
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/operation/ismaster.go:207 +0x18f
go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*Server).check(_)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/server.go:697 +0x457
go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*Server).update(0xc0007d3e00)
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/server.go:505 +0x33f
created by go.mongodb.org/mongo-driver/x/mongo/driver/topology.(*Server).Connect
	/go/pkg/mod/go.mongodb.org/mongo-driver@v1.5.3/x/mongo/driver/topology/server.go:205 +0x171
goroutine 72 [select]:
database/sql.(*DB).connectionOpener(0xc00085e1a0, {0x122cb50, 0xc000850740})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 84 [select]:
database/sql.(*DB).connectionOpener(0xc0003ae820, {0x122cb50, 0xc0000aaec0})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188
rax    0x17
rbx    0xc000312000
rcx    0x4749f5
rdx    0x17
rdi    0x2
rsi    0xfbd901
rbp    0x7f0e722a1530
rsp    0x7f0e72286be0
r8     0x128f
r9     0x1900000
r10    0x0
r11    0x216
r12    0x7f0e721a5800
r13    0x7f0e721a58a0
r14    0xc000102b60
r15    0x7f0e722abd48
rip    0x4730e2
rflags 0x212
cs     0x33
fs     0x0
gs     0x0