runtime: "fatal: morestack on g0" on amd64 after upgrade to Go 1.21

[VirrageS]

Does this issue reproduce with the latest release?

Yes.

What did you do?

https://github.com/VirrageS/fluent-bit-go-bug here is full code to repo.
I will try to make it smaller but for now this is all I have.

When running this project with:

docker build --build-arg="GO_VERSION=1.20" -t fluentbit .
docker run -t fluentbit

it runs correctly.

But when I change to GO_VERSION=1.21 it fails with fatal: morestack on g0:

docker build --build-arg="GO_VERSION=1.21" -t fluentbit .
docker run -t fluentbit

It looks like regression in Go 1.21.
I found similar issues but those seem to be quite old and are not tied to Go 1.21 version.

I have tried to strace the binary and this is what I got:

Details

18    <... set_robust_list resumed>)    = 0
18    prctl(PR_SET_NAME, "flb-pipeline" <unfinished ...>
18    <... prctl resumed>)              = 0
18    futex(0x55c46520b460, FUTEX_WAKE_PRIVATE, 2147483647 <unfinished ...>
18    <... futex resumed>)              = 0
18    mmap(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7f93877fd000
18    munmap(0x7f93877fd000, 2097152 <unfinished ...>
18    <... munmap resumed>)             = 0
18    mmap(NULL, 4190208, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7f93875fe000
18    munmap(0x7f93875fe000, 8192 <unfinished ...>
18    <... munmap resumed>)             = 0
18    munmap(0x7f9387800000, 2084864 <unfinished ...>
18    <... munmap resumed>)             = 0
18    mmap(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0 <unfinished ...>
18    <... mmap resumed>)               = 0x7f9387400000
18    epoll_create1(EPOLL_CLOEXEC)      = 8
18    pipe([9, 10])                     = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 9, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559108056, u64=140272896028120}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    futex(0x55c46520b480, FUTEX_WAKE_PRIVATE, 2147483647) = 0
18    epoll_create1(EPOLL_CLOEXEC <unfinished ...>
18    <... epoll_create1 resumed>)      = 11
18    pipe([12, 13])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 12, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269204480, u64=140271606124544}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    pipe([14, 15])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286400, u64=140271606206464}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    pipe([16, 17])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 16, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286592, u64=140271606206656}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f9386bff000
18    mprotect(0x7f9386c00000, 8388608, PROT_READ|PROT_WRITE|PROT_EXEC <unfinished ...>
18    <... mprotect resumed>)           = 0
18    clone(child_stack=0x7f93873f54f0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID <unfinished ...>
18    <... clone resumed>, parent_tid=[19], tls=0x7f93873ff700, child_tidptr=0x7f93873ff9d0) = 19
18    futex(0x7f93874150a8, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
18    <... futex resumed>)              = 0
18    futex(0x7f93874150b0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 0
18    getpid()                          = 10
18    openat(AT_FDCWD, "/etc/localtime", O_RDONLY|O_CLOEXEC <unfinished ...>
18    <... openat resumed>)             = 18
18    fstat(18,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0644, st_size=118, ...}) = 0
18    fstat(18,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0644, st_size=118, ...}) = 0
18    read(18, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\1\0\0\0\0"..., 4096) = 118
18    lseek(18, -62, SEEK_CUR <unfinished ...>
18    <... lseek resumed>)              = 56
18    read(18, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\1\0\0\0\0"..., 4096) = 62
18    close(18 <unfinished ...>
18    <... close resumed>)              = 0
18    write(15, "T\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    pipe( <unfinished ...>
18    <... pipe resumed>[18, 19])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 18, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559107648, u64=140272896027712}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(15, "m\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    write(15, "7\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    write(15, "7\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 20
18    timerfd_settime(20, TFD_TIMER_ABSTIME, {it_interval={tv_sec=10, tv_nsec=0}, it_value={tv_sec=16839370, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 20, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269405184, u64=140271606325248}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(15, "=\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096) = 4096
18    write(15, "X\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    pipe( <unfinished ...>
18    <... pipe resumed>[21, 22])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 21, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559552000, u64=140272896472064}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    openat(AT_FDCWD, "/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC <unfinished ...>
18    <... openat resumed>)             = 23
18    read(23,  <unfinished ...>
18    <... read resumed>"0-11\n", 8192) = 5
18    close(23)                         = 0
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY <unfinished ...>
18    <... openat resumed>)             = 23
18    fstat(23, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(23,  <unfinished ...>
18    <... read resumed>"cpu  531693645 1691808 251625914"..., 1024) = 1024
18    close(23)                         = 0
18    pipe( <unfinished ...>
18    <... pipe resumed>[23, 24])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 23, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559483392, u64=140272896403456}}) = 0
18    futex(0x55c46520b45c, FUTEX_WAKE_PRIVATE, 2147483647 <unfinished ...>
18    <... futex resumed>)              = 0
18    epoll_create1(EPOLL_CLOEXEC)      = 25
18    pipe( <unfinished ...>
18    <... pipe resumed>[26, 27])       = 0
18    epoll_ctl(25, EPOLL_CTL_ADD, 26, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269552640, u64=140271606472704}}) = 0
18    pipe( <unfinished ...>
18    <... pipe resumed>[28, 29])       = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 28, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286784, u64=140271606206848}}) = 0
18    mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f93863fe000
18    mprotect(0x7f93863ff000, 8388608, PROT_READ|PROT_WRITE|PROT_EXEC <unfinished ...>
18    <... mprotect resumed>)           = 0
18    clone(child_stack=0x7f9386bf44f0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID <unfinished ...>
18    <... clone resumed>, parent_tid=[20], tls=0x7f9386bfe700, child_tidptr=0x7f9386bfe9d0) = 20
18    pipe( <unfinished ...>
18    <... pipe resumed>[30, 31])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 30, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559484160, u64=140272896404224}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    rt_sigprocmask(SIG_SETMASK, NULL,  <unfinished ...>
18    <... rt_sigprocmask resumed>[], 8) = 0
18    rt_sigprocmask(SIG_SETMASK, ~[],  <unfinished ...>
18    <... rt_sigprocmask resumed>NULL, 8) = 0
18    sched_getaffinity(18, 32,  <unfinished ...>
18    <... sched_getaffinity resumed>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) = 8
18    sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0
18    sigaltstack({ss_sp=0xc000180000, ss_flags=0, ss_size=32768},  <unfinished ...>
18    <... sigaltstack resumed>NULL)    = 0
18    rt_sigprocmask(SIG_SETMASK, [],  <unfinished ...>
18    <... rt_sigprocmask resumed>NULL, 8) = 0
18    gettid( <unfinished ...>
18    <... gettid resumed>)             = 18
18    futex(0x7f93d3dcc900, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 1
18    futex(0x7f93d3dcc148, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 1
18    futex(0xc000100d48, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
18    <... futex resumed>)              = -1 EAGAIN (Resource temporarily unavailable)
18    --- SIGURG {si_signo=SIGURG, si_code=SI_TKILL, si_pid=10, si_uid=0} ---
18    rt_sigreturn({mask=[]} <unfinished ...>
18    <... rt_sigreturn resumed>)       = -1 EAGAIN (Resource temporarily unavailable)
18    write(1, "[flb-go] plugin parameter = ''\n", 31 <unfinished ...>
18    <... write resumed>)              = 31
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 36
18    timerfd_settime(36, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 36, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559107912, u64=140272896027976}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 37
18    timerfd_settime(37, TFD_TIMER_ABSTIME, {it_interval={tv_sec=5, tv_nsec=0}, it_value={tv_sec=16839365, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 37, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406144, u64=140271606326208}}) = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 38
18    timerfd_settime(38, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 38, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269396208, u64=140271606316272}}) = 0
18    write(15, "<\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 39
18    timerfd_settime(39, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 39, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269405504, u64=140271606325568}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    timerfd_create(CLOCK_MONOTONIC, 0) = 40
18    timerfd_settime(40, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=500000000}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 40, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406304, u64=140271606326368}}) = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 41
18    timerfd_settime(41, TFD_TIMER_ABSTIME, {it_interval={tv_sec=0, tv_nsec=250000000}, it_value={tv_sec=16839360, tv_nsec=0}}, NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 41, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406464, u64=140271606326528}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(7, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[], 256, 0) = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=2269406304, u64=140271606326368}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 5
18    read(40,  <unfinished ...>
18    <... read resumed>"\1\0\0\0\0\0\0\0", 8) = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, 0) = 4
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}], 256, 0) = 3
18    read(36, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 2
18    read(38, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 1
18    read(39, "\1\0\0\0\0\0\0\0", 8)   = 8
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY) = 42
18    fstat(42, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(42, "cpu  531693681 1691808 251625926"..., 1024) = 1024
18    close(42)                         = 0
18    getpid()                          = 10
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 4
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 3
18    read(36, "\1\0\0\0\0\0\0\0", 8)   = 8
18    write(27, "@fD\207\223\177\0\0", 8) = 8
18    write(10, "P5E\207\223\177\0\0", 8 <unfinished ...>
18    <... write resumed>)              = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559108056, u64=140272896028120}}], 256, 0) = 3
18    read(38, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559108056, u64=140272896028120}}], 256, 0) = 2
18    read(39, "\1\0\0\0\0\0\0\0", 8)   = 8
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY <unfinished ...>
18    <... openat resumed>)             = 42
18    fstat(42,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(42,  <unfinished ...>
18    <... read resumed>"cpu  531693722 1691808 251625947"..., 1024) = 1024
18    close(42 <unfinished ...>
18    <... close resumed>)              = 0
18    getpid( <unfinished ...>
18    <... getpid resumed>)             = 10
18    epoll_wait(8, [{EPOLLIN, {u32=3559108056, u64=140272896028120}}, {EPOLLIN, {u32=3559483392, u64=140272896403456}}], 256, 0) = 2
18    read(9, "P5E\207\223\177\0\0", 8) = 8
18    write(2, "fatal: morestack on g0\n", 23) = 23
18    --- SIGTRAP {si_signo=SIGTRAP, si_code=SI_KERNEL} ---
18    +++ killed by SIGTRAP (core dumped) +++

What did you expect to see?

Same behavior as with GO_VERSION=1.20.

What did you see instead?

fatal: morestack on g0

[seankhliao]

I think it was mentioned here #62130 (comment)

[prattmic]

Thanks for the reference @seankhliao. This sounds like the same issue mentioned in that comment, but overall unrelated to #62130. There is no PHP involved here as far as I can tell.

Fluent Bit doesn't seem like the kind of project that would be doing fancy stack manipulation. @VirrageS or @PettitWesley, do you know if Fluent Bit has some sort of non-standard behavior we should know about?

[aarzilli]

Fluent Bit doesn't seem like the kind of project that would be doing fancy stack manipulation

It's in there but I don't know if it's relevant for this particular instance.

[prattmic]

It looks like there is stack manipulation going on.

The stack pointer changes quite a bit between calls:

Thread 8 "flb-pipeline" hit Breakpoint 1, runtime.cgocallback () at /usr/local/go/src/runtime/asm_amd64.s:918
918     /usr/local/go/src/runtime/asm_amd64.s: No such file or directory.
(gdb) i r sp
sp             0x7fe7ce7f30f8      0x7fe7ce7f30f8
(gdb) c
Continuing.
Thread 8 "flb-pipeline" hit Breakpoint 1, runtime.cgocallback () at /usr/local/go/src/runtime/asm_amd64.s:918
918     /usr/local/go/src/runtime/asm_amd64.s: No such file or directory.
(gdb) i r sp
sp             0x7fe7cdad2a38      0x7fe7cdad2a38
(gdb) bt
#0  runtime.cgocallback () at /usr/local/go/src/runtime/asm_amd64.s:918
#1  0x00007fe819480bfd in crosscall2 () at /usr/local/go/src/runtime/cgo/asm_amd64.s:38
#2  0x00007fe8195b8459 in FLBPluginFlush (data=0x7fe7cda8b640, length=1121, tag=0x7fe7cda00180 "cpu.local") at _cgo_export.c:78
#3  0x000055d57a2d9f74 in proxy_go_output_flush (ctx=0x7fe819c23310, data=0x7fe7cda8b640, size=1121, tag=0x7fe7cda13ef0 "cpu.local", tag_len=9) at /src/fluent-bit/src/proxy/go/go.c:142
#4  0x000055d579cc846c in proxy_cb_flush (event_chunk=0x7fe7cda4b320, out_flush=0x7fe7cda53550, i_ins=0x7fe819ca7400, out_context=0x7fe819c23310, config=0x7fe819c3a400)
    at /src/fluent-bit/src/flb_plugin_proxy.c:57
#5  0x000055d579c1fedd in output_pre_cb_flush () at /src/fluent-bit/include/fluent-bit/flb_output.h:590
#6  0x000055d57a7776c7 in co_init () at /src/fluent-bit/lib/monkey/deps/flb_libco/amd64.c:117
#7  0x0000000000000000 in ?? ()

And we can see that the top frame in the stack is something called co_init from a coroutine library used by Fluent Bit: https://github.com/fluent/fluent-bit/blob/master/lib/flb_libco/

So this is effectively the same issue as #62130, though from a completely different library.

[VirrageS]

Fluent Bit doesn't seem like the kind of project that would be doing fancy stack manipulation. @VirrageS or @PettitWesley, do you know if Fluent Bit has some sort of non-standard behavior we should know about?

I'm not really sure :(

It's in there but I don't know if it's relevant for this particular instance.

Looking at the results, most of those seem unrelated, but I might be missing something.

Actually, there is something like https://github.com/fluent/fluent-bit/blob/f7731f3f81b1b32eef36aaa6786d82593d35a841/lib/flb_libco/sjlj.c#L99.

---

I have bisected the problem to ef0dedc. There was already issue with this commit #59678. Maybe there is still something lingering there? Or maybe this commit just uncovered some bug in how fluent-bit works with the stack.

[prattmic]

Thanks for the quick bisect, I was about to suggest ef0dedc (https://go.dev/cl/495855) as the reason this case is suddenly causing problems, even though this use case was already unsupported.

Suppose that these calls always occur on a C-created thread (seems very likely given the Go code is a shared object). Prior to CL 495855, every cgocallback would go to needm and fetch a new M, which it initializes with the current stack. Upon return, the M is dropped and the stack can be freely changed without causing issues for the next call.

After CL 495855, the M is initialized on the first callback with the stack at that time. Upon the next call, the M has been cached and the stack bounds are not updated, so if the stack has been changed we will trip over stack bounds checks.

cc @cherrymui @ianlancetaylor @mknyszek @golang/runtime

Regarding #62130 (comment), it isn't immediately obvious to me why this situation is unworkable. Provided that this is a callback from C-to-Go (with no prior Go components up the stack), I don't see why we should care that the stack bounds of the thread have changed. It seems OK to me that such a callback could re-initialize the stack bounds if they have changed since the last call.

[prattmic]

As an aside, CL 495855 gets the stack bounds from pthread, but this coroutine library doesn't appear to do anything to update pthread's view, so those bounds won't be accurate at all. In the case of this application, the first call appears to not be on a coroutine, so it works OK. It is only later calls that move to a coroutine.

[gopherbot]

Change https://go.dev/cl/525455 mentions this issue: runtime: allow update of system stack bounds

[PettitWesley]

@prattmic Fluent Bit uses simple coroutines that pause and resume, that's all: https://github.com/fluent/fluent-bit/blob/master/DEVELOPER_GUIDE.md#concurrency

https://github.com/fluent/fluent-bit/blob/master/include/fluent-bit/flb_coro.h

I'm not an expert on it unfortunately. @matthewfala can help answer questions. If you can give me specific questions, I can go find the answers from the other maintainers.

[matthewfala]

@prattmic @PettitWesley I'm not too sure how the GO plugins work in Fluent Bit, however for the c plugins, there is "fancy stack manipulation" to get the coroutines to work. On one thread, Fluent Bit swaps out multiple tasks/coroutines to allow for non-blocking async networking with a javascript like event loop. You can see the assembly for the swap function on x86 used below:

https://github.com/fluent/fluent-bit/blob/master/lib/flb_libco/amd64.c#L88-L105

There are similar files for other platforms.

[cherrymui]

Regarding #62130 (comment), it isn't immediately obvious to me why this situation is unworkable. Provided that this is a callback from C-to-Go (with no prior Go components up the stack), I don't see why we should care that the stack bounds of the thread have changed. It seems OK to me that such a callback could re-initialize the stack bounds if they have changed since the last call.

For #62130 (comment), I think it is more about what we (promise to) support. For cgo in general, we probably never really thought about what we support or not if C code switches stacks.

If the C code switches stacks while there is some Go frames on the stack, it may be tricky/difficult. If there is no Go frame at all when the C code switches stacks, it is probably fine. We could set a different g0 stack bounds each time (probably based on the current SP, which would be imprecise, but probably fine).

How big are C coroutine stacks? If they are small (like goroutine stacks), there is a risk of overflow, as the Go runtime generally assumes C stacks are large.

[matthewfala]

Stack size per coroutine is configurable in Fluent Bit: https://github.com/fluent/fluent-bit/blob/master/src/flb_config.c#L152C6-L152C34

Fluent Bit Documentation on setting coro stack size

Stack size depends on platform/os. By default size is:
1.5x to 3x PTHREAD_STACK_MIN (which is the minimum stack size + some buffer.) or the page size ~ whichever is larger.

Calculated by

((3 * STACK_FACTOR * PTHREAD_STACK_MIN) / 2

Please see:
Stack configuration page size - https://github.com/fluent/fluent-bit/blob/master/src/flb_config.c#L293-L298
Stack size default calculation - https://github.com/fluent/fluent-bit/blob/master/include/fluent-bit/flb_coro.h#L64-L78