os, syscall: support ill-formed UTF-16 strings on Windows

Windows UTF-16 strings can contain unpaired surrogates, which can't be
decoded into a valid UTF-8 string. This file defines a set of functions
that can be used to encode and decode potentially ill-formed UTF-16
strings by using the
[the WTF-8 encoding](https://simonsapin.github.io/wtf-8/).

WTF-8 is a strict superset of UTF-8, i.e. any string that is
well-formed in UTF-8 is also well-formed in WTF-8 and the content
is unchanged. Also, the conversion never fails and is lossless.

The benefit of using WTF-8 instead of UTF-8 when decoding a UTF-16
string is that the conversion is lossless even for ill-formed
UTF-16 strings. This property allows to read an ill-formed UTF-16
string, convert it to a Go string, and convert it back to the same
original UTF-16 string.

Fixes #59971

Change-Id: Id6007f6e537844913402b233e73d698688cd5ba6
Reviewed-on: https://go-review.googlesource.com/c/go/+/493036
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Bryan Mills <[email protected]>
Run-TryBot: Quim Muntal <[email protected]>
Reviewed-by: Cherry Mui <[email protected]>
Reviewed-by: Paul Hampson <[email protected]>

Author: qmuntal

src/internal/syscall/execenv/execenv_windows.go

@@ -9,7 +9,6 @@ package execenv
 import (
 	"internal/syscall/windows"
 	"syscall"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -41,7 +40,7 @@ func Default(sys *syscall.SysProcAttr) (env []string, err error) {
 		}
 
 		entry := unsafe.Slice(blockp, (uintptr(end)-uintptr(unsafe.Pointer(blockp)))/2)
-		env = append(env, string(utf16.Decode(entry)))
+		env = append(env, syscall.UTF16ToString(entry))
 		blockp = (*uint16)(unsafe.Add(end, size))
 	}
 	return

src/internal/syscall/windows/registry/value.go

@@ -217,7 +217,7 @@ func (k Key) GetStringsValue(name string) (val []string, valtype uint32, err err
 	from := 0
 	for i, c := range p {
 		if c == 0 {
-			val = append(val, string(utf16.Decode(p[from:i])))
+			val = append(val, syscall.UTF16ToString(p[from:i]))
 			from = i + 1
 		}
 	}

src/internal/syscall/windows/syscall_windows.go

@@ -7,7 +7,6 @@ package windows
 import (
 	"sync"
 	"syscall"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -17,17 +16,13 @@ func UTF16PtrToString(p *uint16) string {
 	if p == nil {
 		return ""
 	}
-	// Find NUL terminator.
 	end := unsafe.Pointer(p)
 	n := 0
 	for *(*uint16)(end) != 0 {
 		end = unsafe.Pointer(uintptr(end) + unsafe.Sizeof(*p))
 		n++
 	}
-	// Turn *uint16 into []uint16.
-	s := unsafe.Slice(p, n)
-	// Decode []uint16 into string.
-	return string(utf16.Decode(s))
+	return syscall.UTF16ToString(unsafe.Slice(p, n))
 }
 
 const (

src/os/dir_windows.go

@@ -11,7 +11,6 @@ import (
 	"runtime"
 	"sync"
 	"syscall"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -104,7 +103,7 @@ func (file *File) readdir(n int, mode readdirMode) (names []string, dirents []Di
 				d.bufp = 0
 			}
 			nameslice := unsafe.Slice(&info.FileName[0], info.FileNameLength/2)
-			name := string(utf16.Decode(nameslice))
+			name := syscall.UTF16ToString(nameslice)
 			if name == "." || name == ".." { // Useless names
 				continue
 			}

src/os/exec/lp_windows_test.go

@@ -587,7 +587,6 @@ package main
 import (
 	"os"
 	"syscall"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -599,7 +598,7 @@ func getMyName() (string, error) {
 	if n == 0 {
 		return "", err
 	}
-	return string(utf16.Decode(b[0:n])), nil
+	return syscall.UTF16ToString(b[0:n]), nil
 }
 
 func main() {

src/os/file_windows.go

@@ -11,7 +11,6 @@ import (
 	"runtime"
 	"sync"
 	"syscall"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -259,7 +258,7 @@ func tempDir() string {
 			// Otherwise remove terminating \.
 			n--
 		}
-		return string(utf16.Decode(b[:n]))
+		return syscall.UTF16ToString(b[:n])
 	}
 }
 

src/os/os_windows_test.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"slices"
 	"sort"
 	"strings"
 	"syscall"
@@ -1377,3 +1378,71 @@ func TestAppExecLinkStat(t *testing.T) {
 		t.Errorf("exec.LookPath(%q) = %q; want %q", pythonPath, p, pythonPath)
 	}
 }
+
+func TestIllformedUTF16FileName(t *testing.T) {
+	dir := t.TempDir()
+	const sep = string(os.PathSeparator)
+	if !strings.HasSuffix(dir, sep) {
+		dir += sep
+	}
+
+	// This UTF-16 file name is ill-formed as it contains low surrogates that are not preceded by high surrogates ([1:5]).
+	namew := []uint16{0x2e, 0xdc6d, 0xdc73, 0xdc79, 0xdc73, 0x30, 0x30, 0x30, 0x31, 0}
+
+	// Create a file whose name contains unpaired surrogates.
+	// Use syscall.CreateFile instead of os.Create to simulate a file that is created by
+	// a non-Go program so the file name hasn't gone through syscall.UTF16FromString.
+	dirw := utf16.Encode([]rune(dir))
+	pathw := append(dirw, namew...)
+	fd, err := syscall.CreateFile(&pathw[0], syscall.GENERIC_ALL, 0, nil, syscall.CREATE_NEW, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	syscall.CloseHandle(fd)
+
+	name := syscall.UTF16ToString(namew)
+	path := filepath.Join(dir, name)
+	// Verify that os.Lstat can query the file.
+	fi, err := os.Lstat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := fi.Name(); got != name {
+		t.Errorf("got %q, want %q", got, name)
+	}
+	// Verify that File.Readdirnames lists the file.
+	f, err := os.Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, err := f.Readdirnames(0)
+	f.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(files, name) {
+		t.Error("file not listed")
+	}
+	// Verify that os.RemoveAll can remove the directory
+	// and that it doesn't hang.
+	err = os.RemoveAll(dir)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestUTF16Alloc(t *testing.T) {
+	allowsPerRun := func(want int, f func()) {
+		t.Helper()
+		got := int(testing.AllocsPerRun(5, f))
+		if got != want {
+			t.Errorf("got %d allocs, want %d", got, want)
+		}
+	}
+	allowsPerRun(1, func() {
+		syscall.UTF16ToString([]uint16{'a', 'b', 'c'})
+	})
+	allowsPerRun(1, func() {
+		syscall.UTF16FromString("abc")
+	})
+}

src/syscall/env_windows.go

@@ -7,7 +7,6 @@
 package syscall
 
 import (
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -24,7 +23,7 @@ func Getenv(key string) (value string, found bool) {
 			return "", false
 		}
 		if n <= uint32(len(b)) {
-			return string(utf16.Decode(b[:n])), true
+			return UTF16ToString(b[:n]), true
 		}
 	}
 }
@@ -90,7 +89,7 @@ func Environ() []string {
 		}
 
 		entry := unsafe.Slice(envp, (uintptr(end)-uintptr(unsafe.Pointer(envp)))/size)
-		r = append(r, string(utf16.Decode(entry)))
+		r = append(r, UTF16ToString(entry))
 		envp = (*uint16)(unsafe.Add(end, size))
 	}
 	return r

src/syscall/export_windows_test.go

@@ -9,3 +9,6 @@ var UpdateProcThreadAttribute = updateProcThreadAttribute
 var DeleteProcThreadAttributeList = deleteProcThreadAttributeList
 
 const PROC_THREAD_ATTRIBUTE_HANDLE_LIST = _PROC_THREAD_ATTRIBUTE_HANDLE_LIST
+
+var EncodeWTF16 = encodeWTF16
+var DecodeWTF16 = decodeWTF16

src/syscall/syscall_windows.go

@@ -14,7 +14,6 @@ import (
 	"internal/race"
 	"runtime"
 	"sync"
-	"unicode/utf16"
 	"unsafe"
 )
 
@@ -37,7 +36,8 @@ func StringToUTF16(s string) []uint16 {
 
 // UTF16FromString returns the UTF-16 encoding of the UTF-8 string
 // s, with a terminating NUL added. If s contains a NUL byte at any
-// location, it returns (nil, EINVAL).
+// location, it returns (nil, EINVAL). Unpaired surrogates
+// are encoded using WTF-8.
 func UTF16FromString(s string) ([]uint16, error) {
 	if bytealg.IndexByteString(s, 0) != -1 {
 		return nil, EINVAL
@@ -49,22 +49,37 @@ func UTF16FromString(s string) ([]uint16, error) {
 	// equal than the number of UTF-16 code units.
 	// Also account for the terminating NUL character.
 	buf := make([]uint16, 0, len(s)+1)
-	for _, r := range s {
-		buf = utf16.AppendRune(buf, r)
-	}
-	return utf16.AppendRune(buf, '\x00'), nil
+	buf = encodeWTF16(s, buf)
+	return append(buf, 0), nil
 }
 
 // UTF16ToString returns the UTF-8 encoding of the UTF-16 sequence s,
-// with a terminating NUL removed.
+// with a terminating NUL removed. Unpaired surrogates are decoded
+// using WTF-8 instead of UTF-8 encoding.
 func UTF16ToString(s []uint16) string {
+	maxLen := 0
 	for i, v := range s {
 		if v == 0 {
 			s = s[0:i]
 			break
 		}
+		switch {
+		case v <= rune1Max:
+			maxLen += 1
+		case v <= rune2Max:
+			maxLen += 2
+		default:
+			// r is a non-surrogate that decodes to 3 bytes,
+			// or is an unpaired surrogate (also 3 bytes in WTF-8),
+			// or is one half of a valid surrogate pair.
+			// If it is half of a pair, we will add 3 for the second surrogate
+			// (total of 6) and overestimate by 2 bytes for the pair,
+			// since the resulting rune only requires 4 bytes.
+			maxLen += 3
+		}
 	}
-	return string(utf16.Decode(s))
+	buf := decodeWTF16(s, make([]byte, 0, maxLen))
+	return unsafe.String(unsafe.SliceData(buf), len(buf))
 }
 
 // utf16PtrToString is like UTF16ToString, but takes *uint16
@@ -73,17 +88,13 @@ func utf16PtrToString(p *uint16) string {
 	if p == nil {
 		return ""
 	}
-	// Find NUL terminator.
 	end := unsafe.Pointer(p)
 	n := 0
 	for *(*uint16)(end) != 0 {
 		end = unsafe.Pointer(uintptr(end) + unsafe.Sizeof(*p))
 		n++
 	}
-	// Turn *uint16 into []uint16.
-	s := unsafe.Slice(p, n)
-	// Decode []uint16 into string.
-	return string(utf16.Decode(s))
+	return UTF16ToString(unsafe.Slice(p, n))
 }
 
 // StringToUTF16Ptr returns pointer to the UTF-16 encoding of
@@ -97,6 +108,7 @@ func StringToUTF16Ptr(s string) *uint16 { return &StringToUTF16(s)[0] }
 // UTF16PtrFromString returns pointer to the UTF-16 encoding of
 // the UTF-8 string s, with a terminating NUL added. If s
 // contains a NUL byte at any location, it returns (nil, EINVAL).
+// Unpaired surrogates are encoded using WTF-8.
 func UTF16PtrFromString(s string) (*uint16, error) {
 	a, err := UTF16FromString(s)
 	if err != nil {
@@ -143,7 +155,7 @@ func (e Errno) Error() string {
 	// trim terminating \r and \n
 	for ; n > 0 && (b[n-1] == '\n' || b[n-1] == '\r'); n-- {
 	}
-	return string(utf16.Decode(b[:n]))
+	return UTF16ToString(b[:n])
 }
 
 const (
@@ -525,7 +537,7 @@ func Getwd() (wd string, err error) {
 	if e != nil {
 		return "", e
 	}
-	return string(utf16.Decode(b[0:n])), nil
+	return UTF16ToString(b[0:n]), nil
 }
 
 func Chdir(path string) (err error) {
@@ -573,13 +585,13 @@ func Rename(oldpath, newpath string) (err error) {
 }
 
 func ComputerName() (name string, err error) {
-	var n uint32 = MAX_COMPUTERNAME_LENGTH + 1
-	b := make([]uint16, n)
+	b := make([]uint16, MAX_COMPUTERNAME_LENGTH+1)
+	var n uint32
 	e := GetComputerName(&b[0], &n)
 	if e != nil {
 		return "", e
 	}
-	return string(utf16.Decode(b[0:n])), nil
+	return UTF16ToString(b[:n]), nil
 }
 
 func Ftruncate(fd Handle, length int64) (err error) {