Merge pull request #41389 from Azoy/fix-indic-sequences

[stdlib] Fix backwards count of Indic graphemes

Author: Azoy

stdlib/private/StdlibUnicodeUnittest/CMakeLists.txt

@@ -6,6 +6,7 @@ add_swift_target_library(swiftStdlibUnicodeUnittest ${SWIFT_STDLIB_LIBRARY_BUILD
   StdlibUnicodeUnittest.swift
   Collation.swift
   UnicodeScalarProperties.swift
+  GraphemeBreaking.swift
 
   SWIFT_MODULE_DEPENDS StdlibUnittest
   SWIFT_MODULE_DEPENDS_LINUX Glibc

stdlib/private/StdlibUnicodeUnittest/GraphemeBreaking.swift

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// Normalization tests are currently only avaible on Darwin, awaiting a sensible
+// file API...
+#if _runtime(_ObjC)
+import Foundation
+
+func parseGraphemeBreakTests(
+  _ data: String,
+  into result: inout [(String, Int)]
+) {
+  for line in data.split(separator: "\n") {
+    // Only look at actual tests
+    guard line.hasPrefix("÷") else {
+      continue
+    }
+
+    let info = line.split(separator: "#")
+    let components = info[0].split(separator: " ")
+
+    var string = ""
+    var count = 0
+
+    for i in components.indices {
+      guard i != 0 else {
+        continue
+      }
+
+      let scalar: Unicode.Scalar
+
+      // If we're an odd index, this is a scalar.
+      if i & 0x1 == 1 {
+        scalar = Unicode.Scalar(UInt32(components[i], radix: 16)!)!
+
+        string.unicodeScalars.append(scalar)
+      } else {
+        // Otherwise, it is a grapheme breaking operator.
+
+        // If this is a break, record the +1 count. Otherwise it is × which is
+        // not a break.
+        if components[i] == "÷" {
+          count += 1
+        }
+      }
+    }
+
+    result.append((string, count))
+  }
+}
+
+public let graphemeBreakTests: [(String, Int)] = {
+  var result: [(String, Int)] = []
+
+  let testFile = readInputFile("GraphemeBreakTest.txt")
+
+  parseGraphemeBreakTests(testFile, into: &result)
+
+  return result
+}()
+#endif

stdlib/public/core/StringGraphemeBreaking.swift

@@ -390,7 +390,8 @@ extension _StringGuts {
       // If we're currently in an indic sequence (or if our lhs is a linking
       // consonant), then this check and everything underneath ensures that
       // we continue being in one and may check if this extend is a Virama.
-      if state.isInIndicSequence || scalar1._isLinkingConsonant {
+      if state.isInIndicSequence ||
+         (!isBackwards && scalar1._isLinkingConsonant) {
         if y == .extend {
           let extendNormData = Unicode._NormData(scalar2, fastUpperbound: 0x300)
 
@@ -440,7 +441,8 @@ extension _StringGuts {
     // GB999
     default:
       // GB9c
-      if state.isInIndicSequence, state.hasSeenVirama, scalar2._isLinkingConsonant {
+      if !isBackwards, state.isInIndicSequence, state.hasSeenVirama,
+         scalar2._isLinkingConsonant {
         state.hasSeenVirama = false
         return false
       }