math/big: avoid negative slice size in nat.rem

In a division, normally the answer to N digits / D digits has N-D digits,
but not when N-D is negative. Fix the calculation of the number of
digits for the temporary in nat.rem not to be negative.

Fixes #72043.

Change-Id: Ib9faa430aeb6c5f4c4a730f1ec631d2bf3f7472c
Reviewed-on: https://go-review.googlesource.com/c/go/+/655156
Reviewed-by: Robert Griesemer <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: rsc

src/math/big/alias_test.go

@@ -309,4 +309,28 @@ func TestAliasing(t *testing.T) {
 			}
 		})
 	}
+
+	t.Run("Issue72043a", func(t *testing.T) {
+		v := new(big.Int).SetBytes([]byte{0x4b, 0xd2, 0x7a, 0x6c, 0xb3, 0x16, 0x5a, 0xf3, 0xa0, 0x45, 0x2b, 0xbe, 0x38, 0xba, 0x18, 0xb8, 0x70, 0xbb, 0xde, 0xd2, 0x20, 0xe9, 0xb9, 0x9d, 0x1d, 0xc6, 0x3e, 0xa, 0xa5, 0x60, 0x11, 0xcc, 0xb8, 0xff, 0x8e, 0x62})
+		x := new(big.Int).SetBytes([]byte{0xb, 0x40, 0x70, 0x8b, 0x64, 0x78, 0x29, 0x9e, 0xd0})
+		y := new(big.Int).SetBytes([]byte{0x1})
+		z := new(big.Int).SetBytes([]byte{0xe, 0x91, 0x4d, 0xaa, 0x69, 0x60, 0x51, 0x42, 0x9, 0xa1, 0xf6, 0xfa, 0x5, 0x97, 0x1c, 0x5e, 0x18, 0xaa, 0x91, 0x87, 0x1f, 0xe4, 0x70, 0x84, 0x54, 0x97, 0x3a, 0xc9, 0xe1, 0x92, 0xd4, 0xf3, 0xdf, 0x79, 0x90, 0x18, 0xdd, 0xdf, 0x19, 0x44, 0xf3, 0xf, 0x9d, 0xa6, 0xf, 0xd9})
+		v.Exp(x, y, z)
+	})
+
+	t.Run("Issue72043b", func(t *testing.T) {
+		v := new(big.Int).SetBytes([]byte{0x1, 0x6b, 0x8a, 0x26, 0x2, 0x1d, 0x3b, 0x4, 0x8d, 0x81, 0x85, 0xdb, 0x78, 0x4e, 0x81, 0xb7, 0xa7, 0xc0, 0x6e, 0xe7, 0xa0, 0xa, 0x54, 0x39, 0x3e, 0xa8, 0xb1, 0xe0, 0x0, 0xdb, 0x5a, 0x45, 0xde, 0xbf, 0xe2, 0xc8, 0x4d, 0x2b, 0x80, 0x65, 0x73, 0x5d, 0x6d, 0x65, 0x40, 0x50, 0xd4, 0x8})
+		x := new(big.Int).SetBytes([]byte{0x7, 0xd6, 0x79, 0xc5, 0x30, 0xc2, 0x1f, 0x39, 0x5e, 0x3d, 0xd2, 0x52, 0xc6, 0xfb, 0xa7, 0xe2, 0x8a, 0x37, 0xdd})
+		y := new(big.Int).SetBytes([]byte{0x1})
+		z := new(big.Int).SetBytes([]byte{0x1c, 0x44, 0xba, 0x75, 0xfa, 0xe5, 0x3a, 0x56, 0xc0, 0x94, 0xa, 0x1e, 0x8f, 0xbd, 0xad, 0xdb, 0x17, 0xb8, 0x2b, 0x46, 0xf8, 0x69, 0xc1, 0x54, 0x25, 0x68, 0x53, 0xb5, 0xd5, 0x89, 0x6c, 0x16, 0xef, 0xf2, 0x83, 0x3e, 0x55, 0xa7, 0x54, 0x32, 0xbd, 0xa7, 0x70, 0x13, 0xa4, 0xfa})
+		v.Exp(x, y, z)
+	})
+
+	t.Run("Issue72043c", func(t *testing.T) {
+		v := new(big.Int).SetBytes([]byte{0x2, 0x2a, 0xfc, 0x65, 0xc2, 0x66, 0xb1, 0x63, 0x56, 0xaf, 0x5c, 0x6, 0x4a, 0x19, 0x82, 0x6b, 0xaa, 0x90, 0x57, 0xd9, 0xd0, 0x3e, 0x83, 0xc8, 0x2a, 0x85, 0xe0})
+		x := new(big.Int).SetBytes([]byte{0x8, 0x6f, 0xc5, 0x74, 0x83, 0x56, 0xa5, 0x6e, 0xfe, 0x7b, 0x32, 0xfb, 0x69, 0x4, 0x3, 0x51, 0xf5, 0x2c, 0x41, 0x7, 0xf7, 0xbb, 0x89, 0x8e})
+		y := new(big.Int).SetBytes([]byte{0x1})
+		z := new(big.Int).SetBytes([]byte{0x34, 0xe1, 0x5f, 0xd0, 0xa7, 0xf8, 0x34, 0x7c, 0x3c, 0x63, 0x82, 0x1e, 0xc3, 0x84, 0x9f, 0xa, 0x70, 0x22, 0xd3, 0xc8, 0x24, 0x64, 0x2b, 0xd7, 0x59, 0x20, 0xf2, 0xdd, 0xb5, 0xb2, 0xfa, 0x11, 0x2e, 0x98, 0x95})
+		v.Exp(x, y, z)
+	})
 }

src/math/big/natdiv.go

@@ -507,7 +507,7 @@ func (z nat) rem(stk *stack, u, v nat) (r nat) {
 		z = nil
 	}
 	defer stk.restore(stk.save())
-	q := stk.nat(len(u) - (len(v) - 1))
+	q := stk.nat(max(1, len(u)-(len(v)-1)))
 	_, r = q.div(stk, z, u, v)
 	return r
 }