sync/atomic: public And/Or apis

Fixes [reserved]

Author: mauri870

api/next/61335.txt

@@ -0,0 +1,21 @@
+pkg sync/atomic, func AndInt32(*int32, int32) int32 #61395
+pkg sync/atomic, func AndInt64(*int64, int64) int64 #61395
+pkg sync/atomic, func AndUint32(*uint32, uint32) uint32 #61395
+pkg sync/atomic, func AndUint64(*uint64, uint64) uint64 #61395
+pkg sync/atomic, func AndUintptr(*uintptr, uintptr) uintptr #61395
+pkg sync/atomic, func OrInt32(*int32, int32) int32 #61395
+pkg sync/atomic, func OrInt64(*int64, int64) int64 #61395
+pkg sync/atomic, func OrUint32(*uint32, uint32) uint32 #61395
+pkg sync/atomic, func OrUint64(*uint64, uint64) uint64 #61395
+pkg sync/atomic, func OrUintptr(*uintptr, uintptr) uintptr #61395
+pkg sync/atomic, method (*Int32) And(int32) int32 #61395
+pkg sync/atomic, method (*Int64) And(int64) int64 #61395
+pkg sync/atomic, method (*Uint32) And(uint32) uint32 #61395
+pkg sync/atomic, method (*Uint64) And(uint64) uint64 #61395
+pkg sync/atomic, method (*Uintptr) And(uintptr) uintptr #61395
+pkg sync/atomic, method (*Int32) Or(int32) int32 #61395
+pkg sync/atomic, method (*Int64) Or(int64) int64 #61395
+pkg sync/atomic, method (*Uint32) Or(uint32) uint32 #61395
+pkg sync/atomic, method (*Uint64) Or(uint64) uint64 #61395
+pkg sync/atomic, method (*Uintptr) Or(uintptr) uintptr #61395
+

src/runtime/race.go

@@ -323,6 +323,10 @@ var __tsan_report_count byte
 //go:cgo_import_static __tsan_go_atomic64_exchange
 //go:cgo_import_static __tsan_go_atomic32_fetch_add
 //go:cgo_import_static __tsan_go_atomic64_fetch_add
+//go:cgo_import_static __tsan_go_atomic32_fetch_and
+//go:cgo_import_static __tsan_go_atomic64_fetch_and
+//go:cgo_import_static __tsan_go_atomic32_fetch_or
+//go:cgo_import_static __tsan_go_atomic64_fetch_or
 //go:cgo_import_static __tsan_go_atomic32_compare_exchange
 //go:cgo_import_static __tsan_go_atomic64_compare_exchange
 
@@ -642,6 +646,36 @@ func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64)
 //go:linkname abigen_sync_atomic_AddUintptr sync/atomic.AddUintptr
 func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr)
 
+//go:linkname abigen_sync_atomic_AndInt32 sync/atomic.AndInt32
+func abigen_sync_atomic_AndInt32(addr *int32, mask int32) (old int32)
+
+//go:linkname abigen_sync_atomic_AndUint32 sync/atomic.AndUint32
+func abigen_sync_atomic_AndUint32(addr *uint32, mask uint32) (old uint32)
+
+//go:linkname abigen_sync_atomic_AndInt64 sync/atomic.AndInt64
+func abigen_sync_atomic_AndInt64(addr *int64, mask int64) (old int64)
+
+//go:linkname abigen_sync_atomic_AndUint64 sync/atomic.AndUint64
+func abigen_sync_atomic_AndUint64(addr *uint64, mask uint64) (old uint64)
+
+//go:linkname abigen_sync_atomic_AndUintptr sync/atomic.AndUintptr
+func abigen_sync_atomic_AndUintptr(addr *uintptr, mask uintptr) (old uintptr)
+
+//go:linkname abigen_sync_atomic_OrInt32 sync/atomic.OrInt32
+func abigen_sync_atomic_OrInt32(addr *int32, mask int32) (old int32)
+
+//go:linkname abigen_sync_atomic_OrUint32 sync/atomic.OrUint32
+func abigen_sync_atomic_OrUint32(addr *uint32, mask uint32) (old uint32)
+
+//go:linkname abigen_sync_atomic_OrInt64 sync/atomic.OrInt64
+func abigen_sync_atomic_OrInt64(addr *int64, mask int64) (old int64)
+
+//go:linkname abigen_sync_atomic_OrUint64 sync/atomic.OrUint64
+func abigen_sync_atomic_OrUint64(addr *uint64, mask uint64) (old uint64)
+
+//go:linkname abigen_sync_atomic_OrUintptr sync/atomic.OrUintptr
+func abigen_sync_atomic_OrUintptr(addr *uintptr, mask uintptr) (old uintptr)
+
 //go:linkname abigen_sync_atomic_CompareAndSwapInt32 sync/atomic.CompareAndSwapInt32
 func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
 

src/runtime/race_amd64.s

@@ -162,7 +162,7 @@ data:
 	JAE	ret
 call:
 	MOVQ	AX, AX		// w/o this 6a miscompiles this function
-	JMP	racecall<>(SB)
+	JMP	racecall(SB)
 ret:
 	RET
 
@@ -180,8 +180,8 @@ TEXT	racefuncenter<>(SB), NOSPLIT|NOFRAME, $0-0
 	MOVQ	R11, RARG1
 	// void __tsan_func_enter(ThreadState *thr, void *pc);
 	MOVQ	$__tsan_func_enter(SB), AX
-	// racecall<> preserves BX
-	CALL	racecall<>(SB)
+	// racecall preserves BX
+	CALL	racecall(SB)
 	MOVQ	BX, DX	// restore function entry context
 	RET
 
@@ -191,7 +191,7 @@ TEXT	runtime·racefuncexit(SB), NOSPLIT, $0-0
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
 	// void __tsan_func_exit(ThreadState *thr);
 	MOVQ	$__tsan_func_exit(SB), AX
-	JMP	racecall<>(SB)
+	JMP	racecall(SB)
 
 // Atomic operations for sync/atomic package.
 
@@ -303,6 +303,59 @@ TEXT	sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24
 	GO_ARGS
 	JMP	sync∕atomic·AddInt64(SB)
 
+
+// For OpenBSD, see race_openbsd_amd64.s for implementation.
+#ifndef GOOS_openbsd
+// And
+TEXT	sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20
+	GO_ARGS
+	MOVQ	$__tsan_go_atomic32_fetch_and(SB), AX
+	CALL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24
+	GO_ARGS
+	MOVQ	$__tsan_go_atomic64_fetch_and(SB), AX
+	CALL	racecallatomic<>(SB)
+	RET
+// Or
+TEXT	sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20
+	GO_ARGS
+	MOVQ	$__tsan_go_atomic32_fetch_or(SB), AX
+	CALL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24
+	GO_ARGS
+	MOVQ	$__tsan_go_atomic64_fetch_or(SB), AX
+	CALL	racecallatomic<>(SB)
+	RET
+#endif
+
+TEXT	sync∕atomic·AndUint32(SB), NOSPLIT, $0-20
+	GO_ARGS
+	JMP	sync∕atomic·AndInt32(SB)
+
+TEXT	sync∕atomic·AndUint64(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·AndInt64(SB)
+
+TEXT	sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·AndInt64(SB)
+
+TEXT	sync∕atomic·OrUint32(SB), NOSPLIT, $0-20
+	GO_ARGS
+	JMP	sync∕atomic·OrInt32(SB)
+
+TEXT	sync∕atomic·OrUint64(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·OrInt64(SB)
+
+TEXT	sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·OrInt64(SB)
+
 // CompareAndSwap
 TEXT	sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT|NOFRAME, $0-17
 	GO_ARGS
@@ -350,26 +403,26 @@ racecallatomic_ok:
 	MOVQ	8(SP), RARG1	// caller pc
 	MOVQ	(SP), RARG2	// pc
 	LEAQ	16(SP), RARG3	// arguments
-	JMP	racecall<>(SB)	// does not return
+	JMP	racecall(SB)	// does not return
 racecallatomic_ignore:
 	// Addr is outside the good range.
 	// Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.
 	// An attempt to synchronize on the address would cause crash.
 	MOVQ	AX, BX	// remember the original function
 	MOVQ	$__tsan_go_ignore_sync_begin(SB), AX
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
-	CALL	racecall<>(SB)
+	CALL	racecall(SB)
 	MOVQ	BX, AX	// restore the original function
 	// Call the atomic function.
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
 	MOVQ	8(SP), RARG1	// caller pc
 	MOVQ	(SP), RARG2	// pc
 	LEAQ	16(SP), RARG3	// arguments
-	CALL	racecall<>(SB)
+	CALL	racecall(SB)
 	// Call __tsan_go_ignore_sync_end.
 	MOVQ	$__tsan_go_ignore_sync_end(SB), AX
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
-	JMP	racecall<>(SB)
+	JMP	racecall(SB)
 
 // void runtime·racecall(void(*f)(...), ...)
 // Calls C function f from race runtime and passes up to 4 arguments to it.
@@ -380,10 +433,10 @@ TEXT	runtime·racecall(SB), NOSPLIT, $0-0
 	MOVQ	arg1+16(FP), RARG1
 	MOVQ	arg2+24(FP), RARG2
 	MOVQ	arg3+32(FP), RARG3
-	JMP	racecall<>(SB)
+	JMP	racecall(SB)
 
 // Switches SP to g0 stack and calls (AX). Arguments already set.
-TEXT	racecall<>(SB), NOSPLIT|NOFRAME, $0-0
+TEXT	racecall(SB), NOSPLIT|NOFRAME, $0-0
 	MOVQ	g_m(R14), R13
 	// Switch to g0 stack.
 	MOVQ	SP, R12		// callee-saved, preserved across the CALL

src/runtime/race_arm64.s

@@ -312,6 +312,56 @@ TEXT	sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24
 	GO_ARGS
 	JMP	sync∕atomic·AddInt64(SB)
 
+// And
+TEXT	sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20
+	GO_ARGS
+	MOVD	$__tsan_go_atomic32_fetch_and(SB), R9
+	BL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24
+	GO_ARGS
+	MOVD	$__tsan_go_atomic64_fetch_and(SB), R9
+	BL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·AndUint32(SB), NOSPLIT, $0-20
+	GO_ARGS
+	JMP	sync∕atomic·AndInt32(SB)
+
+TEXT	sync∕atomic·AndUint64(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·AndInt64(SB)
+
+TEXT	sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·AndInt64(SB)
+
+// Or
+TEXT	sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20
+	GO_ARGS
+	MOVD	$__tsan_go_atomic32_fetch_or(SB), R9
+	BL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24
+	GO_ARGS
+	MOVD	$__tsan_go_atomic64_fetch_or(SB), R9
+	BL	racecallatomic<>(SB)
+	RET
+
+TEXT	sync∕atomic·OrUint32(SB), NOSPLIT, $0-20
+	GO_ARGS
+	JMP	sync∕atomic·OrInt32(SB)
+
+TEXT	sync∕atomic·OrUint64(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·OrInt64(SB)
+
+TEXT	sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24
+	GO_ARGS
+	JMP	sync∕atomic·OrInt64(SB)
+
 // CompareAndSwap
 TEXT	sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17
 	GO_ARGS

src/runtime/race_openbsd_amd64.s

@@ -0,0 +1,110 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build race
+
+// OpenBSD uses tsan v2, which don't have access to the tsan v3 apis.
+// This file implements the missing operations on top of the v2 apis.
+
+#include "go_asm.h"
+#include "funcdata.h"
+#include "textflag.h"
+
+#define RARG0 DI
+#define RARG1 SI
+#define RARG2 DX
+#define RARG3 CX
+
+TEXT	sync∕atomic·AndInt64(SB), NOSPLIT, $0-24
+	GO_ARGS
+
+	MOVQ 	addr+0(FP), R12
+	MOVQ 	mask+8(FP), R13
+	MOVQ 	(R12), R15
+	ANDQ 	R15, R13
+
+	// args for tsan cas
+	MOVQ 	R12, 16(SP)	// addr
+	MOVQ 	R15, 24(SP)	// old
+	MOVQ 	R13, 32(SP)	// new
+
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	MOVQ	8(SP), RARG1			// caller pc
+	MOVQ	(SP), RARG2				// pc
+	LEAQ	16(SP), RARG3			// arguments
+	MOVQ	$__tsan_go_atomic64_compare_exchange(SB), AX
+	CALL	racecall(SB)
+
+	MOVQ	R15, ret+16(FP)
+
+	RET
+
+TEXT	sync∕atomic·AndInt32(SB), NOSPLIT, $0-20
+	GO_ARGS
+
+	MOVQ 	addr+0(FP), R12
+	MOVL 	mask+8(FP), R13
+	MOVL 	(R12), R15
+	ANDL 	R15, R13
+
+	// args for tsan cas
+	MOVQ 	R12, 16(SP)	// addr
+	MOVL 	R15, 24(SP)	// old
+	MOVL 	R13, 28(SP)	// new
+
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	MOVQ	8(SP), RARG1	// caller pc
+	MOVQ	(SP), RARG2	// pc
+	LEAQ	16(SP), RARG3	// arguments
+	MOVQ	$__tsan_go_atomic32_compare_exchange(SB), AX
+	CALL	racecall(SB)	
+
+	MOVQ	R15, ret+16(FP)
+	RET
+
+// Or
+TEXT	sync∕atomic·OrInt32(SB), NOSPLIT, $0-20
+	GO_ARGS
+
+	MOVQ 	addr+0(FP), R12
+	MOVL 	mask+8(FP), R13
+	MOVL 	(R12), R15
+	ORL 	R15, R13
+
+	MOVQ 	R12, 16(SP)	// addr
+	MOVL 	R15, 24(SP)	// old
+	MOVL 	R13, 28(SP)	// new
+
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	MOVQ	8(SP), RARG1	// caller pc
+	MOVQ	(SP), RARG2	// pc
+	LEAQ	16(SP), RARG3	// arguments
+	MOVQ	$__tsan_go_atomic32_compare_exchange(SB), AX
+	CALL	racecall(SB)
+
+	MOVQ	R15, ret+16(FP)
+	RET
+
+TEXT	sync∕atomic·OrInt64(SB), NOSPLIT, $0-24
+	GO_ARGS
+
+	MOVQ 	addr+0(FP), R12
+	MOVQ 	mask+8(FP), R13
+	MOVQ 	(R12), R15
+	ORQ 	R15, R13
+
+	// args for tsan cas
+	MOVQ 	R12, 16(SP)	// addr
+	MOVQ 	R15, 24(SP)	// old
+	MOVQ 	R13, 32(SP)	// new
+
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	MOVQ	8(SP), RARG1	// caller pc
+	MOVQ	(SP), RARG2	// pc
+	LEAQ	16(SP), RARG3	// arguments
+	MOVQ	$__tsan_go_atomic64_compare_exchange(SB), AX
+	CALL	racecall(SB)
+
+	MOVQ	R15, ret+16(FP)
+	RET