runtime: r2 corrupted by mid-stack inlining on ppc64le

[mkumatag]

What version of Go are you using (go version)?

$ go version
go version go1.12rc1 linux/ppc64le

Does this issue reproduce with the latest release?

Yes, reproduced with 1.12rc1

What operating system and processor architecture are you using (go env)?

go env Output

$ go env
GOARCH="ppc64le"
GOBIN=""
GOCACHE="/root/.cache/go-build"
GOEXE=""
GOFLAGS=""
GOHOSTARCH="ppc64le"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/go"
GOPROXY=""
GORACE=""
GOROOT="/usr/local/go"
GOTMPDIR=""
GOTOOLDIR="/usr/local/go/pkg/tool/linux_ppc64le"
GCCGO="gccgo"
CC="gcc"
CXX="g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build360914728=/tmp/go-build -gno-record-gcc-switches"
#

What did you do?

Steps to reproduce this issues:

root@ip9-114-192-113:~# docker run -ti golang:rc-stretch
root@361515a017ed:/go#
root@361515a017ed:/go# go version
go version go1.12rc1 linux/ppc64le
root@361515a017ed:~# mkdir -p containerd_ws/src/github.com/containerd
root@361515a017ed:~# cd containerd_ws/src/github.com/containerd/
root@361515a017ed:~/containerd_ws/src/github.com/containerd# git clone https://github.com/containerd/containerd.git
Cloning into 'containerd'...
remote: Enumerating objects: 9, done.
remote: Counting objects: 100% (9/9), done.
remote: Compressing objects: 100% (9/9), done.
remote: Total 38383 (delta 2), reused 0 (delta 0), pack-reused 38374
Receiving objects: 100% (38383/38383), 49.63 MiB | 24.02 MiB/s, done.
Resolving deltas: 100% (22575/22575), done.
root@361515a017ed:~/containerd_ws/src/github.com/containerd#
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd# cd metadata/
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# export GOPATH=/root/containerd_ws
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# go test -buildmode pie -v
=== RUN   TestContainersList
unexpected fault address 0x135889140
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x135889140 pc=0x135dbd820]

goroutine 19 [running]:
runtime.throw(0x13634f828, 0x5)
	/usr/local/go/src/runtime/panic.go:617 +0x68 fp=0xc0001fb0f0 sp=0xc0001fb0b0 pc=0x135ddff38
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:397 +0x464 fp=0xc0001fb130 sp=0xc0001fb0f0 pc=0x135df8d64
runtime.mapiterinit(0x135889100, 0xc00020e360, 0xc0001fb328)
	/usr/local/go/src/runtime/map.go:823 +0xa0 fp=0xc0001fb180 sp=0xc0001fb150 pc=0x135dbd820
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb400, 0xc00020e300, 0xc0001fb530)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:526 +0x80 fp=0xc0001fb388 sp=0xc0001fb180 pc=0x13600eb40
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb380, 0xc00020e200, 0xc0001fb738)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb590 sp=0xc0001fb388 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb300, 0xc00020e200, 0xc0001fb940)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb798 sp=0xc0001fb590 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb280, 0xc00020e100, 0xc0001fbb48)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb9a0 sp=0xc0001fb798 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0001fe1d8, 0x62c845, 0x136b6a680)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fbba8 sp=0xc0001fb9a0 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Tx).Commit(0xc0001fe1c0, 0x0, 0x0)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/tx.go:160 +0xd0 fp=0xc0001fbce8 sp=0xc0001fbba8 pc=0x13601c1e0
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*DB).Update(0xc00019a000, 0xc0001fbed8, 0x0, 0x0)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/db.go:677 +0x108 fp=0xc0001fbd48 sp=0xc0001fbce8 pc=0x136013ef8
github.com/containerd/containerd/metadata.TestContainersList(0xc0000f4400)
	/root/containerd_ws/src/github.com/containerd/containerd/metadata/containers_test.go:74 +0x4e0 fp=0xc0001fbf70 sp=0xc0001fbd48 pc=0x13631b2d0
testing.tRunner(0xc0000f4400, 0x1366ca600)
	/usr/local/go/src/testing/testing.go:865 +0xdc fp=0xc0001fbfb0 sp=0xc0001fbf70 pc=0x135eb1dfc
runtime.goexit()
	/usr/local/go/src/runtime/asm_ppc64x.s:856 +0x4 fp=0xc0001fbfb0 sp=0xc0001fbfb0 pc=0x135e135f4
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x304

goroutine 1 [chan receive]:
testing.(*T).Run(0xc0000f4400, 0x136359315, 0x12, 0x1366ca600, 0x136b27f00)
	/usr/local/go/src/testing/testing.go:917 +0x320
testing.runTests.func1(0xc0000f4300)
	/usr/local/go/src/testing/testing.go:1157 +0x8c
testing.tRunner(0xc0000f4300, 0xc0000d3dd8)
	/usr/local/go/src/testing/testing.go:865 +0xdc
testing.runTests(0xc000193660, 0x136b29600, 0x10, 0x10, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x2a0
testing.(*M).Run(0xc0000f2280, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x174
main.main()
	_testmain.go:74 +0x150
exit status 2
FAIL	github.com/containerd/containerd/metadata	0.013s
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#

root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# uname -a
Linux 361515a017ed 4.4.0-141-generic #167-Ubuntu SMP Wed Dec 5 10:33:00 UTC 2018 ppc64le GNU/Linux
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
NAME="Debian GNU/Linux"
VERSION_ID="9"
VERSION="9 (stretch)"
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# lscpu
Architecture:          ppc64le
Byte Order:            Little Endian
CPU(s):                16
On-line CPU(s) list:   0-15
Thread(s) per core:    8
Core(s) per socket:    1
Socket(s):             2
NUMA node(s):          2
Model:                 2.1 (pvr 004b 0201)
Model name:            POWER8 (architected), altivec supported
L1d cache:             64K
L1i cache:             32K
NUMA node0 CPU(s):     0-15
NUMA node2 CPU(s):
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#

What did you expect to see?

No panic

What did you see instead?

Seen panic with fatal error: fault error

[mkumatag]

/cc @laboger

[mvdan]

Does this happen with 1.11.5? If not, this is a regression that needs attention soon.

[mkumatag]

Does this happen with 1.11.5? If not, this is a regression that needs attention soon.

@mvdan tests are passing without any issues in 1.11.5 version, test results are attached here - https://gist.github.com/mkumatag/24f2068a803f36758f938da206312d9f

[odeke-em]

Thank you for the report @mkumatag and @mvdan for the initial response.

@mkumatag, initially I couldn't reproduce your produce using the Docker container steps as some steps seem to be missing. However, after a plain:

go get github.com/containerd/containerd && \
cd $GOPATH/src/github.com/containerd/containerd/metadata && \
go test -buildmode=pie -v

I cannot get a crash either on the Docker container nor on my Macbook pro, where both are
using Go1.12rc1 and the instructions I've provided can be run on both the Docker container and on your computer.

[mkumatag]

Thank you for the report @mkumatag and @mvdan for the initial response.

@mkumatag, initially I couldn't reproduce your produce using the Docker container steps as some steps seem to be missing. However, after a plain:

go get github.com/containerd/containerd && \
cd $GOPATH/src/github.com/containerd/containerd/metadata && \
go test -buildmode=pie -v

I cannot get a crash either on the Docker container nor on my Macbook pro, where both are
using Go1.12rc1 and the instructions I've provided can be run on both the Docker container and on your computer.

This is happening in the ppc64le architecture, lemme modify the description

[thaJeztah]

Just from looking at the location where this was failing (see moby/moby#38404 (comment)), these changes could be possible suspects: 05166bf and 020a18c

/cc @martisch

[laboger]

I have found that it starts to fail with commit 69c2c56. @randall77

The SEGV occurs because r2 has been corrupted during a call to github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable from github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill.

Before this change, the code at the end of github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable looks like this:

  599bf4:       08 00 05 e9     ld      r8,8(r5)
  599bf8:       08 00 08 e9     ld      r8,8(r8)
  599bfc:       10 00 e7 38     addi    r7,r7,16
  599c00:       80 00 08 e9     ld      r8,128(r8)
  599c04:       76 fe 09 7d     sradi   r9,r8,63
  599c08:       a0 17 29 79     rldicl  r9,r9,2,62
  599c0c:       14 4a 08 7d     add     r8,r8,r9
  599c10:       74 16 08 7d     sradi   r8,r8,2
  599c14:       00 40 27 7c     cmpd    r7,r8
  599c18:       24 00 81 41     bgt     599c3c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x16c>
  599c1c:       28 00 01 e9     ld      r8,40(r1)
  599c20:       01 00 08 39     addi    r8,r8,1
  599c24:       00 30 28 7c     cmpd    r8,r6
  599c28:       14 ff 80 41     blt     599b3c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x6c>
  599c2c:       01 00 60 38     li      r3,1
  599c30:       e0 00 61 98     stb     r3,224(r1)
  599c34:       b8 00 21 38     addi    r1,r1,184
  599c38:       20 00 80 4e     blr
  599c3c:       e0 00 01 98     stb     r0,224(r1)
  599c40:       b8 00 21 38     addi    r1,r1,184
  599c44:       20 00 80 4e     blr
  599c48:       e0 00 01 98     stb     r0,224(r1)
  599c4c:       b8 00 21 38     addi    r1,r1,184
  599c50:       20 00 80 4e     blr
  599c54:       e0 00 01 98     stb     r0,224(r1)
  599c58:       b8 00 21 38     addi    r1,r1,184
  599c5c:       20 00 80 4e     blr

After this change, the same code looks like this. It is incorrectly loading the value of r2 from 24(r1). That location has never been initialized with the value of r2 so some random value is being loaded and then used by the code when this function returns.

After this change, the same code looks like this:
  5aa1c4:       08 00 05 e9     ld      r8,8(r5)
  5aa1c8:       08 00 08 e9     ld      r8,8(r8)
  5aa1cc:       10 00 e7 38     addi    r7,r7,16
  5aa1d0:       78 03 00 7c     mr      r0,r0
  5aa1d4:       18 00 41 e8     ld      r2,24(r1)  <----- This load is wrong, r2 has never been saved here.
  5aa1d8:       80 00 08 e9     ld      r8,128(r8)
  5aa1dc:       76 fe 09 7d     sradi   r9,r8,63
  5aa1e0:       a0 17 29 79     rldicl  r9,r9,2,62
  5aa1e4:       14 4a 08 7d     add     r8,r8,r9
  5aa1e8:       74 16 08 7d     sradi   r8,r8,2
  5aa1ec:       00 40 27 7c     cmpd    r7,r8
  5aa1f0:       24 00 81 41     bgt     5aa214 <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x174>
  5aa1f4:       28 00 01 e9     ld      r8,40(r1)
  5aa1f8:       01 00 08 39     addi    r8,r8,1
  5aa1fc:       00 30 28 7c     cmpd    r8,r6
  5aa200:       0c ff 80 41     blt     5aa10c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x6c>
  5aa204:       01 00 60 38     li      r3,1
  5aa208:       e0 00 61 98     stb     r3,224(r1)
  5aa20c:       b8 00 21 38     addi    r1,r1,184
  5aa210:       20 00 80 4e     blr
  5aa214:       e0 00 01 98     stb     r0,224(r1)
  5aa218:       b8 00 21 38     addi    r1,r1,184
  5aa21c:       20 00 80 4e     blr
  5aa220:       e0 00 01 98     stb     r0,224(r1)
  5aa224:       b8 00 21 38     addi    r1,r1,184
  5aa228:       20 00 80 4e     blr
  5aa22c:       e0 00 01 98     stb     r0,224(r1)
  5aa230:       b8 00 21 38     addi    r1,r1,184
  5aa234:       20 00 80 4e     blr

[laboger]

FYI... I was able to reproduce the problem without building it in a container.