cmd/compile: avoid infinite recursion when inlining closures

CL 630696 changes budget for once-called closures, making them more
inlinable. However, when recursive inlining involve both the closure and
its parent, the inliner goes into an infinite loop:

	parent (a closure)  -> closure -> parent -> ...

The problem here dues to the closure name mangling, causing the inlined
checking condition failed, since the closure name affects how the
linker symbol generated.

To fix this, just prevent the closure from inlining its parent into
itself, avoid the infinite inlining loop.

Fixes #71680

Change-Id: Ib27626d70f95e5f1c24a3eb1c8e6c3443b7d90c8
Reviewed-on: https://go-review.googlesource.com/c/go/+/649656
Reviewed-by: David Chase <[email protected]>
Auto-Submit: Cuong Manh Le <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Michael Knyszek <[email protected]>

Author: cuonglm

src/cmd/compile/internal/inline/inl.go

@@ -1009,6 +1009,38 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 		return false, 0, false
 	}
 
+	isClosureParent := func(closure, parent *ir.Func) bool {
+		for p := closure.ClosureParent; p != nil; p = p.ClosureParent {
+			if p == parent {
+				return true
+			}
+		}
+		return false
+	}
+	if isClosureParent(callerfn, callee) {
+		// Can't recursively inline a parent of the closure into itself.
+		if log && logopt.Enabled() {
+			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to closure parent: %s, %s", ir.FuncName(callerfn), ir.FuncName(callee)))
+		}
+		return false, 0, false
+	}
+	if isClosureParent(callee, callerfn) {
+		// Can't recursively inline a closure if there's a call to the parent in closure body.
+		if ir.Any(callee, func(node ir.Node) bool {
+			if call, ok := node.(*ir.CallExpr); ok {
+				if name, ok := call.Fun.(*ir.Name); ok && isClosureParent(callerfn, name.Func) {
+					return true
+				}
+			}
+			return false
+		}) {
+			if log && logopt.Enabled() {
+				logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to closure parent: %s, %s", ir.FuncName(callerfn), ir.FuncName(callee)))
+			}
+			return false, 0, false
+		}
+	}
+
 	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
 		// Runtime package must not be instrumented.
 		// Instrument skips runtime package. However, some runtime code can be

test/fixedbugs/issue71680.go

@@ -0,0 +1,28 @@
+// compile
+
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type Parser struct{}
+type Node struct{}
+
+type parserState func(p *Parser) parserState
+
+func parserStateData(root *Node) parserState {
+	return func(p *Parser) parserState {
+		return parserStateOpenMap(root)(p)
+	}
+}
+
+func parserStateOpenMap(root *Node) parserState {
+	return func(p *Parser) parserState {
+		switch {
+		case p != nil:
+			return parserStateData(root)(p)
+		}
+		return parserStateOpenMap(root)(p)
+	}
+}