[Runtime] Properly handle tuple types in layout string instantiation (#66603)

Tuple types contain metadata entries for each element, so we can handle them individually to avoid unnecessary indirection.

Author: drexin

include/swift/Runtime/Metadata.h

@@ -666,9 +666,6 @@ void swift_initStructMetadataWithLayoutString(StructMetadata *self,
                                               const uint8_t *fieldTags,
                                               uint32_t *fieldOffsets);
 
-SWIFT_RUNTIME_STDLIB_INTERNAL
-size_t _swift_refCountBytesForMetatype(const Metadata *type);
-
 enum LayoutStringFlags : uint64_t {
   Empty = 0,
   // TODO: Track other useful information tha can be used to optimize layout
@@ -688,6 +685,9 @@ inline LayoutStringFlags &operator|=(LayoutStringFlags &a, LayoutStringFlags b)
   return a = (a | b);
 }
 
+SWIFT_RUNTIME_STDLIB_INTERNAL
+size_t _swift_refCountBytesForMetatype(const Metadata *type);
+
 SWIFT_RUNTIME_STDLIB_INTERNAL
 void _swift_addRefCountStringForMetatype(uint8_t *layoutStr,
                                          size_t &layoutStrOffset,

stdlib/public/runtime/BytecodeLayouts.cpp

@@ -37,26 +37,6 @@
 
 using namespace swift;
 
-static const size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t);
-
-/// Given a pointer and an offset, read the requested data and increment the
-/// offset
-template <typename T>
-static T readBytes(const uint8_t *typeLayout, size_t &i) {
-  T returnVal;
-  memcpy(&returnVal, typeLayout + i, sizeof(T));
-  i += sizeof(T);
-  return returnVal;
-}
-
-/// Given a pointer, a value, and an offset, write the value at the given
-/// offset and increment offset by the size of T
-template <typename T>
-static void writeBytes(uint8_t *typeLayout, size_t &i, T value) {
-  memcpy(typeLayout + i, &value, sizeof(T));
-  i += sizeof(T);
-}
-
 static Metadata *getExistentialTypeMetadata(OpaqueValue *object) {
   return reinterpret_cast<Metadata**>(object)[NumWords_ValueBuffer];
 }

stdlib/public/runtime/BytecodeLayouts.h

@@ -73,6 +73,23 @@ void swift_resolve_resilientAccessors(uint8_t *layoutStr,
                                       size_t layoutStrOffset,
                                       const uint8_t *fieldLayoutStr,
                                       const Metadata *fieldType);
+
+template <typename T>
+inline T readBytes(const uint8_t *layoutStr, size_t &i) {
+  T returnVal;
+  memcpy(&returnVal, layoutStr + i, sizeof(T));
+  i += sizeof(T);
+  return returnVal;
+}
+
+template <typename T>
+inline void writeBytes(uint8_t *layoutStr, size_t &i, T value) {
+  memcpy(layoutStr + i, &value, sizeof(T));
+  i += sizeof(T);
+}
+
+constexpr size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t);
+
 } // namespace swift
 
 #endif // SWIFT_BYTECODE_LAYOUTS_H

stdlib/public/runtime/Metadata.cpp

@@ -2641,41 +2641,6 @@ void swift::swift_initStructMetadata(StructMetadata *structType,
   vwtable->publishLayout(layout);
 }
 
-namespace {
-  enum LayoutStringFlags : uint64_t {
-    Empty = 0,
-    // TODO: Track other useful information tha can be used to optimize layout
-    //       strings, like different reference kinds contained in the string
-    //       number of ref counting operations (maybe up to 4), so we can
-    //       use witness functions optimized for these cases.
-    HasRelativePointers = (1ULL << 63),
-  };
-
-  inline bool operator&(LayoutStringFlags a, LayoutStringFlags b) {
-    return (uint64_t(a) & uint64_t(b)) != 0;
-  }
-  inline LayoutStringFlags operator|(LayoutStringFlags a, LayoutStringFlags b) {
-    return LayoutStringFlags(uint64_t(a) | uint64_t(b));
-  }
-  inline LayoutStringFlags &operator|=(LayoutStringFlags &a, LayoutStringFlags b) {
-    return a = (a | b);
-  }
-
-  template <typename T>
-  inline T readBytes(const uint8_t *layoutStr, size_t &i) {
-    T returnVal;
-    memcpy(&returnVal, layoutStr + i, sizeof(T));
-    i += sizeof(T);
-    return returnVal;
-  }
-
-  template <typename T>
-  inline void writeBytes(uint8_t *layoutStr, size_t &i, T value) {
-    memcpy(layoutStr + i, &value, sizeof(T));
-    i += sizeof(T);
-  }
-} // end anonymous namespace
-
 void swift::swift_initStructMetadataWithLayoutString(
     StructMetadata *structType, StructLayoutFlags layoutFlags, size_t numFields,
     const uint8_t *const *fieldTypes, const uint8_t *fieldTags,
@@ -2727,21 +2692,9 @@ void swift::swift_initStructMetadataWithLayoutString(
       extraInhabitantCount = fieldExtraInhabitantCount;
     }
 
-    if (fieldType->vw_size() == 0) {
-      continue;
-    } else if (fieldType->getValueWitnesses()->isPOD()) {
-      // no extra space required for POD
-    } else if (fieldType->hasLayoutString()) {
-      refCountBytes += *(const size_t *)(fieldType->getLayoutString() +
-                                         sizeof(uint64_t));
-    } else if (fieldType->isClassObject() || fieldType->isAnyExistentialType()) {
-      refCountBytes += sizeof(uint64_t);
-    } else {
-      refCountBytes += sizeof(uint64_t) + sizeof(uintptr_t);
-    }
+    refCountBytes += _swift_refCountBytesForMetatype(fieldType);
   }
 
-  const size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t);
   const size_t fixedLayoutStringSize = layoutStringHeaderSize +
                                        sizeof(uint64_t) * 2;
 
@@ -2781,93 +2734,9 @@ void swift::swift_initStructMetadataWithLayoutString(
 
     const Metadata *fieldType = (const Metadata*)fieldTypes[i];
 
-    fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1);
-    size_t offset = fullOffset - unalignedOffset + previousFieldOffset;
-
-    if (fieldType->vw_size() == 0) {
-      continue;
-    } else if (fieldType->getValueWitnesses()->isPOD()) {
-      // No need to handle PODs
-      previousFieldOffset = offset + fieldType->vw_size();
-      fullOffset += fieldType->vw_size();
-    } else if (fieldType->hasLayoutString()) {
-      const uint8_t *fieldLayoutStr = fieldType->getLayoutString();
-      const LayoutStringFlags fieldFlags =
-          *(const LayoutStringFlags *)fieldLayoutStr;
-      const size_t fieldRefCountBytes =
-          *(const size_t *)(fieldLayoutStr + sizeof(uint64_t));
-      if (fieldRefCountBytes > 0) {
-        flags |= fieldFlags;
-        memcpy(layoutStr + layoutStrOffset, fieldLayoutStr + layoutStringHeaderSize,
-               fieldRefCountBytes);
-
-        if (fieldFlags & LayoutStringFlags::HasRelativePointers) {
-          swift_resolve_resilientAccessors(layoutStr, layoutStrOffset,
-                                           fieldLayoutStr, fieldType);
-        }
-
-        if (offset) {
-          auto layoutStrOffsetCopy = layoutStrOffset;
-          auto firstTagAndOffset =
-              readBytes<uint64_t>(layoutStr, layoutStrOffsetCopy);
-          layoutStrOffsetCopy = layoutStrOffset;
-          firstTagAndOffset += offset;
-          writeBytes(layoutStr, layoutStrOffsetCopy, firstTagAndOffset);
-        }
-
-        auto previousFieldOffsetOffset =
-            layoutStringHeaderSize + fieldRefCountBytes;
-        previousFieldOffset = readBytes<uint64_t>(fieldLayoutStr,
-                                                  previousFieldOffsetOffset);
-        layoutStrOffset += fieldRefCountBytes;
-      } else {
-        previousFieldOffset += fieldType->vw_size();
-      }
-      fullOffset += fieldType->vw_size();
-    } else if (auto *cls = fieldType->getClassObject()) {
-      RefCountingKind tag;
-      if (!cls->isTypeMetadata()) {
-      #if SWIFT_OBJC_INTEROP
-        tag = RefCountingKind::ObjC;
-      #else
-        tag = RefCountingKind::Unknown;
-      #endif
-      } else {
-        auto *vwt = cls->getValueWitnesses();
-        if (vwt == &VALUE_WITNESS_SYM(Bo)) {
-          tag = RefCountingKind::NativeStrong;
-        } else if (vwt == &VALUE_WITNESS_SYM(BO)) {
-        #if SWIFT_OBJC_INTEROP
-          tag = RefCountingKind::ObjC;
-        #else
-          tag = RefCountingKind::Unknown;
-        #endif
-        } else if (vwt == &VALUE_WITNESS_SYM(Bb)) {
-          tag = RefCountingKind::Bridge;
-        } else {
-          goto metadata;
-        };
-      }
-
-      writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset);
-      previousFieldOffset = fieldType->vw_size();
-      fullOffset += previousFieldOffset;
-    } else if (fieldType->isAnyExistentialType()) {
-      auto *existential = dyn_cast<ExistentialTypeMetadata>(fieldType);
-      assert(existential);
-      auto tag = existential->isClassBounded() ? RefCountingKind::Unknown
-                                               : RefCountingKind::Existential;
-      writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset);
-      previousFieldOffset = fieldType->vw_size();
-      fullOffset += previousFieldOffset;
-    } else {
-metadata:
-      writeBytes(layoutStr, layoutStrOffset,
-                 ((uint64_t)RefCountingKind::Metatype << 56) | offset);
-      writeBytes(layoutStr, layoutStrOffset, fieldType);
-      previousFieldOffset = fieldType->vw_size();
-      fullOffset += previousFieldOffset;
-    }
+    _swift_addRefCountStringForMetatype(layoutStr, layoutStrOffset, flags,
+                                        fieldType, fullOffset,
+                                        previousFieldOffset);
   }
 
   writeBytes(layoutStr, layoutStrOffset, previousFieldOffset);
@@ -2896,6 +2765,128 @@ void swift::swift_initStructMetadataWithLayoutString(
   vwtable->publishLayout(layout);
 }
 
+size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) {
+  if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) {
+    return 0;
+  } else if (type->hasLayoutString()) {
+    size_t offset = sizeof(uint64_t);
+    return readBytes<size_t>(type->getLayoutString(), offset);
+  } else if (type->isClassObject() || type->isAnyExistentialType()) {
+    return sizeof(uint64_t);
+  } else if (auto *tuple = dyn_cast<TupleTypeMetadata>(type)) {
+    size_t res = 0;
+    for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) {
+      res += _swift_refCountBytesForMetatype(tuple->getElement(i).Type);
+    }
+    return res;
+  } else {
+    return sizeof(uint64_t) + sizeof(uintptr_t);
+  }
+}
+
+void swift::_swift_addRefCountStringForMetatype(uint8_t *layoutStr,
+                                                size_t &layoutStrOffset,
+                                                LayoutStringFlags &flags,
+                                                const Metadata *fieldType,
+                                                size_t &fullOffset,
+                                                size_t &previousFieldOffset) {
+  size_t unalignedOffset = fullOffset;
+  fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1);
+  size_t offset = fullOffset - unalignedOffset + previousFieldOffset;
+  if (fieldType->vw_size() == 0) {
+    return;
+  } else if (fieldType->getValueWitnesses()->isPOD()) {
+    // No need to handle PODs
+    previousFieldOffset = offset + fieldType->vw_size();
+    fullOffset += fieldType->vw_size();
+  } else if (fieldType->hasLayoutString()) {
+    const uint8_t *fieldLayoutStr = fieldType->getLayoutString();
+    const LayoutStringFlags fieldFlags =
+        *(const LayoutStringFlags *)fieldLayoutStr;
+    size_t refCountBytesOffset = sizeof(uint64_t);
+    const size_t fieldRefCountBytes = readBytes<size_t>(fieldLayoutStr,
+                                                        refCountBytesOffset);
+    if (fieldRefCountBytes > 0) {
+      flags |= fieldFlags;
+      memcpy(layoutStr + layoutStrOffset,
+             fieldLayoutStr + layoutStringHeaderSize,
+             fieldRefCountBytes);
+
+      if (fieldFlags & LayoutStringFlags::HasRelativePointers) {
+        swift_resolve_resilientAccessors(layoutStr, layoutStrOffset,
+                                         fieldLayoutStr, fieldType);
+      }
+
+      if (offset) {
+        auto layoutStrOffsetCopy = layoutStrOffset;
+        auto firstTagAndOffset =
+            readBytes<uint64_t>(layoutStr, layoutStrOffsetCopy);
+        layoutStrOffsetCopy = layoutStrOffset;
+        firstTagAndOffset += offset;
+        writeBytes(layoutStr, layoutStrOffsetCopy, firstTagAndOffset);
+      }
+
+      auto previousFieldOffsetOffset =
+          layoutStringHeaderSize + fieldRefCountBytes;
+      previousFieldOffset = readBytes<uint64_t>(fieldLayoutStr,
+                                                previousFieldOffsetOffset);
+      layoutStrOffset += fieldRefCountBytes;
+    } else {
+      previousFieldOffset += fieldType->vw_size();
+    }
+    fullOffset += fieldType->vw_size();
+  } else if (auto *tuple = dyn_cast<TupleTypeMetadata>(fieldType)) {
+    for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) {
+      _swift_addRefCountStringForMetatype(layoutStr, layoutStrOffset, flags,
+                                          tuple->getElement(i).Type, fullOffset,
+                                          previousFieldOffset);
+    }
+  } else if (auto *cls = fieldType->getClassObject()) {
+    RefCountingKind tag;
+    if (!cls->isTypeMetadata()) {
+    #if SWIFT_OBJC_INTEROP
+      tag = RefCountingKind::ObjC;
+    #else
+      tag = RefCountingKind::Unknown;
+    #endif
+    } else {
+      auto *vwt = cls->getValueWitnesses();
+      if (vwt == &VALUE_WITNESS_SYM(Bo)) {
+        tag = RefCountingKind::NativeStrong;
+      } else if (vwt == &VALUE_WITNESS_SYM(BO)) {
+      #if SWIFT_OBJC_INTEROP
+        tag = RefCountingKind::ObjC;
+      #else
+        tag = RefCountingKind::Unknown;
+      #endif
+      } else if (vwt == &VALUE_WITNESS_SYM(Bb)) {
+        tag = RefCountingKind::Bridge;
+      } else {
+        goto metadata;
+      };
+    }
+
+    writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset);
+    previousFieldOffset = fieldType->vw_size();
+    fullOffset += previousFieldOffset;
+  } else if (fieldType->isAnyExistentialType()) {
+    auto *existential = dyn_cast<ExistentialTypeMetadata>(fieldType);
+    assert(existential);
+    auto tag = existential->isClassBounded() ? RefCountingKind::Unknown
+                                             : RefCountingKind::Existential;
+    writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset);
+    previousFieldOffset = fieldType->vw_size();
+    fullOffset += previousFieldOffset;
+  } else {
+metadata:
+    writeBytes(layoutStr, layoutStrOffset,
+               ((uint64_t)RefCountingKind::Metatype << 56) | offset);
+    writeBytes(layoutStr, layoutStrOffset, fieldType);
+    previousFieldOffset = fieldType->vw_size();
+    fullOffset += previousFieldOffset;
+  }
+}
+
 /***************************************************************************/
 /*** Classes ***************************************************************/
 /***************************************************************************/