Remove wrong include

libc/src/string/memory_utils/op_generic.h

@@ -141,52 +141,42 @@ template <typename T> struct Memset {
   static_assert(is_element_type_v<T>);
   static constexpr size_t SIZE = sizeof(T);
 
-  LIBC_INLINE static void block(Ptr dst, uint8_t value) {
+  LIBC_INLINE static void block_offset(Ptr dst, uint8_t value, size_t offset) {
     if constexpr (is_scalar_v<T> || is_vector_v<T>) {
-      store<T>(dst, splat<T>(value));
+      store<T>(dst + offset, splat<T>(value));
     } else if constexpr (is_array_v<T>) {
       using value_type = typename T::value_type;
       const auto Splat = splat<value_type>(value);
       for (size_t I = 0; I < array_size_v<T>; ++I)
-        store<value_type>(dst + (I * sizeof(value_type)), Splat);
+        store<value_type>(dst + offset + (I * sizeof(value_type)), Splat);
     }
   }
 
+  LIBC_INLINE static void block(Ptr dst, uint8_t value) {
+    block_offset(dst, value, 0);
+  }
+
   LIBC_INLINE static void tail(Ptr dst, uint8_t value, size_t count) {
-    block(dst + count - SIZE, value);
+    block_offset(dst, value, count - SIZE);
   }
 
   LIBC_INLINE static void head_tail(Ptr dst, uint8_t value, size_t count) {
     block(dst, value);
     tail(dst, value, count);
   }
 
-  LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) {
+  LIBC_INLINE static void loop_and_tail_offset(Ptr dst, uint8_t value,
+                                               size_t count, size_t offset) {
     static_assert(SIZE > 1, "a loop of size 1 does not need tail");
-    size_t offset = 0;
     do {
-      block(dst + offset, value);
+      block_offset(dst, value, offset);
       offset += SIZE;
     } while (offset < count - SIZE);
     tail(dst, value, count);
   }
 
-  template <size_t prefetch_distance, size_t prefetch_degree>
-  LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
-                                                 size_t count) {
-    size_t offset = 96;
-    while (offset + prefetch_degree + SIZE <= count) {
-      for (size_t i = 0; i < prefetch_degree / sw_prefetch::kCachelineSize; ++i)
-        sw_prefetch::PrefetchW(dst + offset + prefetch_distance +
-                               sw_prefetch::kCachelineSize * i);
-      for (size_t i = 0; i < prefetch_degree; i += SIZE, offset += SIZE)
-        block(dst + offset, value);
-    }
-    while (offset + SIZE < count) {
-      block(dst + offset, value);
-      offset += SIZE;
-    }
-    tail(dst, value, count);
+  LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) {
+    return loop_and_tail_offset(dst, value, count, 0);
   }
 };
 

libc/src/string/memory_utils/utils.h

@@ -374,6 +374,12 @@ template <size_t SIZE> struct AlignHelper {
   uintptr_t offset_;
 };
 
+LIBC_INLINE void prefetch_for_write(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
+
+LIBC_INLINE void prefetch_to_local_cache(CPtr dst) {
+  __builtin_prefetch(dst, 0, 3);
+}
+
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_UTILS_H

libc/src/string/memory_utils/x86_64/inline_memcpy.h

@@ -47,11 +47,6 @@ LIBC_INLINE_VAR constexpr size_t kRepMovsbThreshold =
 
 } // namespace x86
 
-// TODO: Move to a shared header when appropriate.
-[[maybe_unused]] LIBC_INLINE void prefetch_to_local_cache(const void *addr) {
-  __builtin_prefetch(addr, 0, 3);
-}
-
 [[maybe_unused]] LIBC_INLINE void
 inline_memcpy_x86_sse2_ge64(Ptr __restrict dst, CPtr __restrict src,
                             size_t count) {

libc/src/string/memory_utils/x86_64/inline_memset.h

@@ -17,78 +17,93 @@
 
 namespace LIBC_NAMESPACE {
 namespace x86 {
+// Size of one cache line for software prefetching
+LIBC_INLINE_VAR constexpr size_t kOneCachelineSize = 64;
+LIBC_INLINE_VAR constexpr size_t kTwoCachelinesSize = kOneCachelineSize * 2;
+LIBC_INLINE_VAR constexpr size_t kFiveCachelinesSize = kOneCachelineSize * 5;
+
 LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
     LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING);
 
 } // namespace x86
 
 #if defined(__AVX512F__)
-  using uint128_t = generic_v128;
-  using uint256_t = generic_v256;
-  using uint512_t = generic_v512;
+using uint128_t = generic_v128;
+using uint256_t = generic_v256;
+using uint512_t = generic_v512;
 #elif defined(__AVX__)
-  using uint128_t = generic_v128;
-  using uint256_t = generic_v256;
-  using uint512_t = cpp::array<generic_v256, 2>;
+using uint128_t = generic_v128;
+using uint256_t = generic_v256;
+using uint512_t = cpp::array<generic_v256, 2>;
 #elif defined(__SSE2__)
-  using uint128_t = generic_v128;
-  using uint256_t = cpp::array<generic_v128, 2>;
-  using uint512_t = cpp::array<generic_v128, 4>;
+using uint128_t = generic_v128;
+using uint256_t = cpp::array<generic_v128, 2>;
+using uint512_t = cpp::array<generic_v128, 4>;
 #else
-  using uint128_t = cpp::array<uint64_t, 2>;
-  using uint256_t = cpp::array<uint64_t, 4>;
-  using uint512_t = cpp::array<uint64_t, 8>;
+using uint128_t = cpp::array<uint64_t, 2>;
+using uint256_t = cpp::array<uint64_t, 4>;
+using uint512_t = cpp::array<uint64_t, 8>;
 #endif
 
-  [[maybe_unused]] LIBC_INLINE static void
-  inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
-    // Prefetch one cacheline
-    sw_prefetch::PrefetchW(dst + sw_prefetch::kCachelineSize);
-    if (count <= 128)
-      return generic::Memset<uint512_t>::head_tail(dst, value, count);
-    // Prefetch the next cacheline
-    sw_prefetch::PrefetchW(dst + sw_prefetch::kCachelineSize * 2);
-    // Aligned loop
-    generic::Memset<uint256_t>::block(dst, value);
-    align_to_next_boundary<32>(dst, count);
-    if (count <= 192) {
-      return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
-    } else {
-      generic::Memset<uint512_t>::block(dst, value);
-      generic::Memset<uint256_t>::block(dst + sizeof(uint512_t), value);
-      return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(
-          dst, value, count);
+[[maybe_unused]] LIBC_INLINE static void
+inline_memset_x86_gt64_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
+  size_t prefetch_distance = x86::kFiveCachelinesSize;
+  size_t prefetch_degree = x86::kTwoCachelinesSize;
+  size_t SIZE = sizeof(uint256_t);
+  // Prefetch one cache line
+  prefetch_for_write(dst + x86::kOneCachelineSize);
+  if (count <= 128)
+    return generic::Memset<uint512_t>::head_tail(dst, value, count);
+  // Prefetch the second cache line
+  prefetch_for_write(dst + x86::kTwoCachelinesSize);
+  // Aligned loop
+  generic::Memset<uint256_t>::block(dst, value);
+  align_to_next_boundary<32>(dst, count);
+  if (count <= 192) {
+    return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
+  } else {
+    generic::Memset<uint512_t>::block(dst, value);
+    generic::Memset<uint256_t>::block_offset(dst, value, SIZE);
+    size_t offset = 96;
+    while (offset + prefetch_degree + SIZE <= count) {
+      for (size_t i = 0; i < prefetch_degree / x86::kOneCachelineSize; ++i)
+        prefetch_for_write(dst + offset + prefetch_distance +
+                           x86::kOneCachelineSize * i);
+      for (size_t i = 0; i < prefetch_degree; i += SIZE, offset += SIZE)
+        generic::Memset<uint256_t>::block_offset(dst, value, offset);
     }
+    generic::Memset<uint256_t>::loop_and_tail_offset(dst, value, count, offset);
   }
+}
+
+[[maybe_unused]] LIBC_INLINE static void
+inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
+  if (count == 0)
+    return;
+  if (count == 1)
+    return generic::Memset<uint8_t>::block(dst, value);
+  if (count == 2)
+    return generic::Memset<uint16_t>::block(dst, value);
+  if (count == 3)
+    return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
+  if (count <= 8)
+    return generic::Memset<uint32_t>::head_tail(dst, value, count);
+  if (count <= 16)
+    return generic::Memset<uint64_t>::head_tail(dst, value, count);
+  if (count <= 32)
+    return generic::Memset<uint128_t>::head_tail(dst, value, count);
+  if (count <= 64)
+    return generic::Memset<uint256_t>::head_tail(dst, value, count);
+  if constexpr (x86::kUseSoftwarePrefetchingMemset)
+    return inline_memset_x86_gt64_sw_prefetching(dst, value, count);
+  if (count <= 128)
+    return generic::Memset<uint512_t>::head_tail(dst, value, count);
+  // Aligned loop
+  generic::Memset<uint256_t>::block(dst, value);
+  align_to_next_boundary<32>(dst, count);
+  return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
+}
 
-  [[maybe_unused]] LIBC_INLINE static void
-  inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
-    if (count == 0)
-      return;
-    if (count == 1)
-      return generic::Memset<uint8_t>::block(dst, value);
-    if (count == 2)
-      return generic::Memset<uint16_t>::block(dst, value);
-    if (count == 3)
-      return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
-    if (count <= 8)
-      return generic::Memset<uint32_t>::head_tail(dst, value, count);
-    if (count <= 16)
-      return generic::Memset<uint64_t>::head_tail(dst, value, count);
-    if (count <= 32)
-      return generic::Memset<uint128_t>::head_tail(dst, value, count);
-    if (count <= 64)
-      return generic::Memset<uint256_t>::head_tail(dst, value, count);
-    if constexpr (x86::kUseSoftwarePrefetchingMemset) {
-      return inline_memset_x86_sw_prefetching(dst, value, count);
-    }
-    if (count <= 128)
-      return generic::Memset<uint512_t>::head_tail(dst, value, count);
-    // Aligned loop
-    generic::Memset<uint256_t>::block(dst, value);
-    align_to_next_boundary<32>(dst, count);
-    return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
-  }
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H