VirtualOutputBackend Improvements for swift compiler

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp

@@ -2907,7 +2907,7 @@ swift::ASTContext *SwiftASTContext::GetASTContext() {
       GetLanguageOptions(), GetTypeCheckerOptions(), GetSILOptions(),
       GetSearchPathOptions(), GetClangImporterOptions(),
       GetSymbolGraphOptions(), GetSourceManager(), GetDiagnosticEngine(),
-      ReportModuleLoadingProgress));
+      /*OutputBackend=*/nullptr, ReportModuleLoadingProgress));
   m_diagnostic_consumer_ap.reset(new StoringDiagnosticConsumer(*this));
 
   if (getenv("LLDB_SWIFT_DUMP_DIAGS")) {

llvm/include/llvm/Support/HashingOutputBackend.h

@@ -0,0 +1,114 @@
+//===- HashingOutputBackends.h - Hashing output backends --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/HashBuilder.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace vfs {
+
+/// raw_pwrite_stream that writes to a hasher.
+template <typename HasherT>
+class HashingStream : public llvm::raw_pwrite_stream {
+private:
+  SmallVector<char> Buffer;
+  raw_svector_ostream OS;
+
+  using HashBuilderT = HashBuilder<HasherT, support::endianness::native>;
+  HashBuilderT Builder;
+
+  void write_impl(const char *Ptr, size_t Size) override {
+    OS.write(Ptr, Size);
+  }
+
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+    OS.pwrite(Ptr, Size, Offset);
+  }
+
+  uint64_t current_pos() const override { return OS.str().size(); }
+
+public:
+  HashingStream() : OS(Buffer) { SetUnbuffered(); }
+
+  auto final() {
+    Builder.update(OS.str());
+    return Builder.final();
+  }
+};
+
+template <typename HasherT> class HashingOutputFile;
+
+/// An output backend that only generates the hash for outputs.
+template <typename HasherT> class HashingOutputBackend : public OutputBackend {
+private:
+  friend class HashingOutputFile<HasherT>;
+  void addOutputFile(StringRef Path, StringRef Hash) {
+    OutputHashes[Path] = std::string(Hash);
+  }
+
+protected:
+  IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+    return const_cast<HashingOutputBackend<HasherT> *>(this);
+  }
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, Optional<OutputConfig> Config) override {
+    return std::make_unique<HashingOutputFile<HasherT>>(Path, *this);
+  }
+
+public:
+  /// Iterator for all the output file names.
+  auto outputFiles() const { return OutputHashes.keys(); }
+
+  /// Get hash value for the output files in hex representation.
+  /// Return None if the requested path is not generated.
+  Optional<std::string> getHashValueForFile(StringRef Path) {
+    auto F = OutputHashes.find(Path);
+    if (F == OutputHashes.end())
+      return None;
+    return toHex(F->second);
+  }
+
+private:
+  StringMap<std::string> OutputHashes;
+};
+
+/// HashingOutputFile.
+template <typename HasherT>
+class HashingOutputFile final : public OutputFileImpl {
+public:
+  Error keep() override {
+    auto Result = OS.final();
+    Backend.addOutputFile(OutputPath, toStringRef(Result));
+    return Error::success();
+  }
+  Error discard() override { return Error::success(); }
+  raw_pwrite_stream &getOS() override { return OS; }
+
+  HashingOutputFile(StringRef OutputPath,
+                    HashingOutputBackend<HasherT> &Backend)
+      : OutputPath(OutputPath.str()), Backend(Backend) {}
+
+private:
+  const std::string OutputPath;
+  HashingStream<HasherT> OS;
+  HashingOutputBackend<HasherT> &Backend;
+};
+
+} // namespace vfs
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H

llvm/include/llvm/Support/VirtualOutputConfig.def

@@ -19,5 +19,7 @@ HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF.
 HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal.
 HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries.
 HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true)
+// Skip atomic write if existing file content is the same
+HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false)
 
 #undef HANDLE_OUTPUT_CONFIG_FLAG

llvm/lib/Support/VirtualOutputBackends.cpp

@@ -337,6 +337,89 @@ Error OnDiskOutputFile::initializeStream() {
   return Error::success();
 }
 
+namespace {
+class OpenFileRAII {
+  static const int InvalidFd = -1;
+
+public:
+  int Fd = InvalidFd;
+
+  ~OpenFileRAII() {
+    if (Fd != InvalidFd)
+      llvm::sys::Process::SafelyCloseFileDescriptor(Fd);
+  }
+};
+
+enum class FileDifference : uint8_t {
+  /// The source and destination paths refer to the exact same file.
+  IdenticalFile,
+  /// The source and destination paths refer to separate files with identical
+  /// contents.
+  SameContents,
+  /// The source and destination paths refer to separate files with different
+  /// contents.
+  DifferentContents
+};
+} // end anonymous namespace
+
+static Expected<FileDifference>
+areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) {
+  if (sys::fs::equivalent(Source, Destination))
+    return FileDifference::IdenticalFile;
+
+  OpenFileRAII SourceFile;
+  sys::fs::file_status SourceStatus;
+  // If we can't open the source file, fail.
+  if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd))
+    return convertToOutputError(Source, EC);
+
+  // If we can't stat the source file, fail.
+  if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus))
+    return convertToOutputError(Source, EC);
+
+  OpenFileRAII DestFile;
+  sys::fs::file_status DestStatus;
+  // If we can't open the destination file, report different.
+  if (std::error_code Error =
+          sys::fs::openFileForRead(Destination, DestFile.Fd))
+    return FileDifference::DifferentContents;
+
+  // If we can't open the destination file, report different.
+  if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus))
+    return FileDifference::DifferentContents;
+
+  // If the files are different sizes, they must be different.
+  uint64_t Size = SourceStatus.getSize();
+  if (Size != DestStatus.getSize())
+    return FileDifference::DifferentContents;
+
+  // If both files are zero size, they must be the same.
+  if (Size == 0)
+    return FileDifference::SameContents;
+
+  // The two files match in size, so we have to compare the bytes to determine
+  // if they're the same.
+  std::error_code SourceRegionErr;
+  sys::fs::mapped_file_region SourceRegion(
+      sys::fs::convertFDToNativeFile(SourceFile.Fd),
+      sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr);
+  if (SourceRegionErr)
+    return convertToOutputError(Source, SourceRegionErr);
+
+  std::error_code DestRegionErr;
+  sys::fs::mapped_file_region DestRegion(
+      sys::fs::convertFDToNativeFile(DestFile.Fd),
+      sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr);
+
+  if (DestRegionErr)
+    return FileDifference::DifferentContents;
+
+  if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0)
+    return FileDifference::DifferentContents;
+
+  return FileDifference::SameContents;
+}
+
 Error OnDiskOutputFile::keep() {
   // Destroy the streams to flush them.
   BufferOS.reset();
@@ -351,6 +434,25 @@ Error OnDiskOutputFile::keep() {
   if (!TempPath)
     return Error::success();
 
+  if (Config.getOnlyIfDifferent()) {
+    auto Result = areFilesDifferent(*TempPath, OutputPath);
+    if (!Result)
+      return Result.takeError();
+    switch (*Result) {
+    case FileDifference::IdenticalFile:
+      // Do nothing for a self-move.
+      return Error::success();
+
+    case FileDifference::SameContents:
+      // Files are identical; remove the source file.
+      (void) sys::fs::remove(*TempPath);
+      return Error::success();
+
+    case FileDifference::DifferentContents:
+      break; // Rename the file.
+    }
+  }
+
   // Move temporary to the final output path and remove it if that fails.
   std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath);
   if (!RenameEC)

llvm/unittests/Support/VirtualOutputBackendsTest.cpp

@@ -8,6 +8,8 @@
 
 #include "llvm/Support/VirtualOutputBackends.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/HashingOutputBackend.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
@@ -765,4 +767,77 @@ TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) {
       FailedWithMessage(Error1->Msg));
 }
 
+TEST(OnDiskBackendTest, OnlyIfDifferent) {
+  OnDiskOutputBackendProvider Provider;
+  auto Backend = Provider.createBackend();
+  std::string FilePath = Provider.getFilePathToCreate();
+  StringRef Data = "some data";
+  OutputConfig Config = OutputConfig().setOnlyIfDifferent();
+
+  OutputFile O1, O2, O3;
+  sys::fs::file_status Status1, Status2, Status3;
+  // Write first file.
+  EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),
+                    Succeeded());
+  O1 << Data;
+  EXPECT_THAT_ERROR(O1.keep(), Succeeded());
+  EXPECT_FALSE(O1.isOpen());
+  EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false));
+
+  // Write second with same content.
+  EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),
+                    Succeeded());
+  O2 << Data;
+  EXPECT_THAT_ERROR(O2.keep(), Succeeded());
+  EXPECT_FALSE(O2.isOpen());
+  EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false));
+
+  // Make sure the output path file is not modified with same content.
+  EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID());
+
+  // Write third with different content.
+  EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3),
+                    Succeeded());
+  O3 << Data << "\n";
+  EXPECT_THAT_ERROR(O3.keep(), Succeeded());
+  EXPECT_FALSE(O3.isOpen());
+  EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false));
+
+  // This should overwrite the file and create a different UniqueID.
+  EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID());
+}
+
+TEST(HashingBackendTest, HashOutput) {
+  HashingOutputBackend<BLAKE3> Backend;
+  OutputFile O1, O2, O3, O4, O5;
+  EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded());
+  O1 << "some data";
+  EXPECT_THAT_ERROR(O1.keep(), Succeeded());
+  EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded());
+  O2 << "some data";
+  EXPECT_THAT_ERROR(O2.keep(), Succeeded());
+  EXPECT_EQ(Backend.getHashValueForFile("file1"),
+            Backend.getHashValueForFile("file2"));
+
+  EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded());
+  O3 << "some ";
+  O3 << "data";
+  EXPECT_THAT_ERROR(O3.keep(), Succeeded());
+  EXPECT_EQ(Backend.getHashValueForFile("file1"),
+            Backend.getHashValueForFile("file3"));
+
+  EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded());
+  O4 << "same data";
+  O4.getOS().pwrite("o", 1, 1);
+  EXPECT_THAT_ERROR(O4.keep(), Succeeded());
+  EXPECT_EQ(Backend.getHashValueForFile("file1"),
+            Backend.getHashValueForFile("file4"));
+
+  EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded());
+  O5 << "different data";
+  EXPECT_THAT_ERROR(O5.keep(), Succeeded());
+  EXPECT_NE(Backend.getHashValueForFile("file1"),
+            Backend.getHashValueForFile("file5"));
+}
+
 } // end namespace

llvm/unittests/Support/VirtualOutputConfigTest.cpp

@@ -22,13 +22,15 @@ TEST(VirtualOutputConfigTest, construct) {
   EXPECT_TRUE(OutputConfig().getDiscardOnSignal());
   EXPECT_TRUE(OutputConfig().getAtomicWrite());
   EXPECT_TRUE(OutputConfig().getImplyCreateDirectories());
+  EXPECT_FALSE(OutputConfig().getOnlyIfDifferent());
 
   // Test inverted defaults.
   EXPECT_TRUE(OutputConfig().getNoText());
   EXPECT_TRUE(OutputConfig().getNoCRLF());
   EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal());
   EXPECT_FALSE(OutputConfig().getNoAtomicWrite());
   EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories());
+  EXPECT_TRUE(OutputConfig().getNoOnlyIfDifferent());
 }
 
 TEST(VirtualOutputConfigTest, set) {