Skip to content

VirtualOutputBackend Improvements for swift compiler #6112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2907,7 +2907,7 @@ swift::ASTContext *SwiftASTContext::GetASTContext() {
GetLanguageOptions(), GetTypeCheckerOptions(), GetSILOptions(),
GetSearchPathOptions(), GetClangImporterOptions(),
GetSymbolGraphOptions(), GetSourceManager(), GetDiagnosticEngine(),
ReportModuleLoadingProgress));
/*OutputBackend=*/nullptr, ReportModuleLoadingProgress));
m_diagnostic_consumer_ap.reset(new StoringDiagnosticConsumer(*this));

if (getenv("LLDB_SWIFT_DUMP_DIAGS")) {
Expand Down
114 changes: 114 additions & 0 deletions llvm/include/llvm/Support/HashingOutputBackend.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
//===- HashingOutputBackends.h - Hashing output backends --------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H

#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualOutputBackend.h"
#include "llvm/Support/VirtualOutputConfig.h"
#include "llvm/Support/raw_ostream.h"

namespace llvm {
namespace vfs {

/// raw_pwrite_stream that writes to a hasher.
template <typename HasherT>
class HashingStream : public llvm::raw_pwrite_stream {
private:
SmallVector<char> Buffer;
raw_svector_ostream OS;

using HashBuilderT = HashBuilder<HasherT, support::endianness::native>;
HashBuilderT Builder;

void write_impl(const char *Ptr, size_t Size) override {
OS.write(Ptr, Size);
}

void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
OS.pwrite(Ptr, Size, Offset);
}

uint64_t current_pos() const override { return OS.str().size(); }

public:
HashingStream() : OS(Buffer) { SetUnbuffered(); }

auto final() {
Builder.update(OS.str());
return Builder.final();
}
};

template <typename HasherT> class HashingOutputFile;

/// An output backend that only generates the hash for outputs.
template <typename HasherT> class HashingOutputBackend : public OutputBackend {
private:
friend class HashingOutputFile<HasherT>;
void addOutputFile(StringRef Path, StringRef Hash) {
OutputHashes[Path] = std::string(Hash);
}

protected:
IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
return const_cast<HashingOutputBackend<HasherT> *>(this);
}

Expected<std::unique_ptr<OutputFileImpl>>
createFileImpl(StringRef Path, Optional<OutputConfig> Config) override {
return std::make_unique<HashingOutputFile<HasherT>>(Path, *this);
}

public:
/// Iterator for all the output file names.
auto outputFiles() const { return OutputHashes.keys(); }

/// Get hash value for the output files in hex representation.
/// Return None if the requested path is not generated.
Optional<std::string> getHashValueForFile(StringRef Path) {
auto F = OutputHashes.find(Path);
if (F == OutputHashes.end())
return None;
return toHex(F->second);
}

private:
StringMap<std::string> OutputHashes;
};

/// HashingOutputFile.
template <typename HasherT>
class HashingOutputFile final : public OutputFileImpl {
public:
Error keep() override {
auto Result = OS.final();
Backend.addOutputFile(OutputPath, toStringRef(Result));
return Error::success();
}
Error discard() override { return Error::success(); }
raw_pwrite_stream &getOS() override { return OS; }

HashingOutputFile(StringRef OutputPath,
HashingOutputBackend<HasherT> &Backend)
: OutputPath(OutputPath.str()), Backend(Backend) {}

private:
const std::string OutputPath;
HashingStream<HasherT> OS;
HashingOutputBackend<HasherT> &Backend;
};

} // namespace vfs
} // namespace llvm

#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
2 changes: 2 additions & 0 deletions llvm/include/llvm/Support/VirtualOutputConfig.def
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,7 @@ HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF.
HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal.
HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries.
HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true)
// Skip atomic write if existing file content is the same
HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false)

#undef HANDLE_OUTPUT_CONFIG_FLAG
102 changes: 102 additions & 0 deletions llvm/lib/Support/VirtualOutputBackends.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,89 @@ Error OnDiskOutputFile::initializeStream() {
return Error::success();
}

namespace {
class OpenFileRAII {
static const int InvalidFd = -1;

public:
int Fd = InvalidFd;

~OpenFileRAII() {
if (Fd != InvalidFd)
llvm::sys::Process::SafelyCloseFileDescriptor(Fd);
}
};

enum class FileDifference : uint8_t {
/// The source and destination paths refer to the exact same file.
IdenticalFile,
/// The source and destination paths refer to separate files with identical
/// contents.
SameContents,
/// The source and destination paths refer to separate files with different
/// contents.
DifferentContents
};
} // end anonymous namespace

static Expected<FileDifference>
areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) {
if (sys::fs::equivalent(Source, Destination))
return FileDifference::IdenticalFile;

OpenFileRAII SourceFile;
sys::fs::file_status SourceStatus;
// If we can't open the source file, fail.
if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd))
return convertToOutputError(Source, EC);

// If we can't stat the source file, fail.
if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus))
return convertToOutputError(Source, EC);

OpenFileRAII DestFile;
sys::fs::file_status DestStatus;
// If we can't open the destination file, report different.
if (std::error_code Error =
sys::fs::openFileForRead(Destination, DestFile.Fd))
return FileDifference::DifferentContents;

// If we can't open the destination file, report different.
if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus))
return FileDifference::DifferentContents;

// If the files are different sizes, they must be different.
uint64_t Size = SourceStatus.getSize();
if (Size != DestStatus.getSize())
return FileDifference::DifferentContents;

// If both files are zero size, they must be the same.
if (Size == 0)
return FileDifference::SameContents;

// The two files match in size, so we have to compare the bytes to determine
// if they're the same.
std::error_code SourceRegionErr;
sys::fs::mapped_file_region SourceRegion(
sys::fs::convertFDToNativeFile(SourceFile.Fd),
sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr);
if (SourceRegionErr)
return convertToOutputError(Source, SourceRegionErr);

std::error_code DestRegionErr;
sys::fs::mapped_file_region DestRegion(
sys::fs::convertFDToNativeFile(DestFile.Fd),
sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr);

if (DestRegionErr)
return FileDifference::DifferentContents;

if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0)
return FileDifference::DifferentContents;

return FileDifference::SameContents;
}

Error OnDiskOutputFile::keep() {
// Destroy the streams to flush them.
BufferOS.reset();
Expand All @@ -351,6 +434,25 @@ Error OnDiskOutputFile::keep() {
if (!TempPath)
return Error::success();

if (Config.getOnlyIfDifferent()) {
auto Result = areFilesDifferent(*TempPath, OutputPath);
if (!Result)
return Result.takeError();
switch (*Result) {
case FileDifference::IdenticalFile:
// Do nothing for a self-move.
return Error::success();

case FileDifference::SameContents:
// Files are identical; remove the source file.
(void) sys::fs::remove(*TempPath);
return Error::success();

case FileDifference::DifferentContents:
break; // Rename the file.
}
}

// Move temporary to the final output path and remove it if that fails.
std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath);
if (!RenameEC)
Expand Down
75 changes: 75 additions & 0 deletions llvm/unittests/Support/VirtualOutputBackendsTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

#include "llvm/Support/VirtualOutputBackends.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/HashingOutputBackend.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"
Expand Down Expand Up @@ -765,4 +767,77 @@ TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) {
FailedWithMessage(Error1->Msg));
}

TEST(OnDiskBackendTest, OnlyIfDifferent) {
OnDiskOutputBackendProvider Provider;
auto Backend = Provider.createBackend();
std::string FilePath = Provider.getFilePathToCreate();
StringRef Data = "some data";
OutputConfig Config = OutputConfig().setOnlyIfDifferent();

OutputFile O1, O2, O3;
sys::fs::file_status Status1, Status2, Status3;
// Write first file.
EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),
Succeeded());
O1 << Data;
EXPECT_THAT_ERROR(O1.keep(), Succeeded());
EXPECT_FALSE(O1.isOpen());
EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false));

// Write second with same content.
EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),
Succeeded());
O2 << Data;
EXPECT_THAT_ERROR(O2.keep(), Succeeded());
EXPECT_FALSE(O2.isOpen());
EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false));

// Make sure the output path file is not modified with same content.
EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID());

// Write third with different content.
EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3),
Succeeded());
O3 << Data << "\n";
EXPECT_THAT_ERROR(O3.keep(), Succeeded());
EXPECT_FALSE(O3.isOpen());
EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false));

// This should overwrite the file and create a different UniqueID.
EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID());
}

TEST(HashingBackendTest, HashOutput) {
HashingOutputBackend<BLAKE3> Backend;
OutputFile O1, O2, O3, O4, O5;
EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded());
O1 << "some data";
EXPECT_THAT_ERROR(O1.keep(), Succeeded());
EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded());
O2 << "some data";
EXPECT_THAT_ERROR(O2.keep(), Succeeded());
EXPECT_EQ(Backend.getHashValueForFile("file1"),
Backend.getHashValueForFile("file2"));

EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded());
O3 << "some ";
O3 << "data";
EXPECT_THAT_ERROR(O3.keep(), Succeeded());
EXPECT_EQ(Backend.getHashValueForFile("file1"),
Backend.getHashValueForFile("file3"));

EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded());
O4 << "same data";
O4.getOS().pwrite("o", 1, 1);
EXPECT_THAT_ERROR(O4.keep(), Succeeded());
EXPECT_EQ(Backend.getHashValueForFile("file1"),
Backend.getHashValueForFile("file4"));

EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded());
O5 << "different data";
EXPECT_THAT_ERROR(O5.keep(), Succeeded());
EXPECT_NE(Backend.getHashValueForFile("file1"),
Backend.getHashValueForFile("file5"));
}

} // end namespace
2 changes: 2 additions & 0 deletions llvm/unittests/Support/VirtualOutputConfigTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@ TEST(VirtualOutputConfigTest, construct) {
EXPECT_TRUE(OutputConfig().getDiscardOnSignal());
EXPECT_TRUE(OutputConfig().getAtomicWrite());
EXPECT_TRUE(OutputConfig().getImplyCreateDirectories());
EXPECT_FALSE(OutputConfig().getOnlyIfDifferent());

// Test inverted defaults.
EXPECT_TRUE(OutputConfig().getNoText());
EXPECT_TRUE(OutputConfig().getNoCRLF());
EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal());
EXPECT_FALSE(OutputConfig().getNoAtomicWrite());
EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories());
EXPECT_TRUE(OutputConfig().getNoOnlyIfDifferent());
}

TEST(VirtualOutputConfigTest, set) {
Expand Down