Skip to content

Commit eccc6e2

Browse files
authored
[clang-repl] Enable extending launchExecutor (#152562)
This patch introduces the ability to customize the fork process with an external lambda function. This is useful for downstream clients where they want to do stream redirection.
1 parent 6a425f1 commit eccc6e2

File tree

5 files changed

+221
-3
lines changed

5 files changed

+221
-3
lines changed

clang/include/clang/Interpreter/RemoteJITUtils.h

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626

2727
llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
2828
launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory,
29-
llvm::StringRef SlabAllocateSizeString);
29+
llvm::StringRef SlabAllocateSizeString,
30+
std::function<void()> CustomizeFork = nullptr);
3031

3132
/// Create a JITLinkExecutor that connects to the given network address
3233
/// through a TCP socket. A valid NetworkAddress provides hostname and port,
@@ -35,4 +36,13 @@ llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
3536
connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory,
3637
llvm::StringRef SlabAllocateSizeString);
3738

39+
#ifdef LLVM_ON_UNIX
40+
/// Returns PID of last launched executor.
41+
pid_t getLastLaunchedExecutorPID();
42+
43+
/// Returns PID of nth launched executor.
44+
/// 1-based indexing.
45+
pid_t getNthLaunchedExecutorPID(int n);
46+
#endif
47+
3848
#endif // LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H

clang/lib/Interpreter/RemoteJITUtils.cpp

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@
3333
using namespace llvm;
3434
using namespace llvm::orc;
3535

36+
#if LLVM_ON_UNIX
37+
static std::vector<pid_t> LaunchedExecutorPID;
38+
#endif
39+
3640
Expected<uint64_t> getSlabAllocSize(StringRef SizeString) {
3741
SizeString = SizeString.trim();
3842

@@ -89,9 +93,14 @@ createSharedMemoryManager(SimpleRemoteEPC &SREPC,
8993
SlabSize, SREPC, SAs);
9094
}
9195

96+
// Launches an out-of-process executor for remote JIT. The calling program can
97+
// provide a CustomizeFork callback, which allows it to run custom code in the
98+
// child process before exec. This enables sending custom setup or code to be
99+
// executed in the child (out-of-process) executor.
92100
Expected<std::unique_ptr<SimpleRemoteEPC>>
93101
launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
94-
llvm::StringRef SlabAllocateSizeString) {
102+
llvm::StringRef SlabAllocateSizeString,
103+
std::function<void()> CustomizeFork) {
95104
#ifndef LLVM_ON_UNIX
96105
// FIXME: Add support for Windows.
97106
return make_error<StringError>("-" + ExecutablePath +
@@ -134,6 +143,9 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
134143
close(ToExecutor[WriteEnd]);
135144
close(FromExecutor[ReadEnd]);
136145

146+
if (CustomizeFork)
147+
CustomizeFork();
148+
137149
// Execute the child process.
138150
std::unique_ptr<char[]> ExecutorPath, FDSpecifier;
139151
{
@@ -158,6 +170,8 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
158170
}
159171
// else we're the parent...
160172

173+
LaunchedExecutorPID.push_back(ChildPID);
174+
161175
// Close the child ends of the pipes
162176
close(ToExecutor[ReadEnd]);
163177
close(FromExecutor[WriteEnd]);
@@ -265,3 +279,18 @@ connectTCPSocket(StringRef NetworkAddress, bool UseSharedMemory,
265279
std::move(S), *SockFD, *SockFD);
266280
#endif
267281
}
282+
283+
#if LLVM_ON_UNIX
284+
285+
pid_t getLastLaunchedExecutorPID() {
286+
if (!LaunchedExecutorPID.size())
287+
return -1;
288+
return LaunchedExecutorPID.back();
289+
}
290+
291+
pid_t getNthLaunchedExecutorPID(int n) {
292+
if (n - 1 < 0 || n - 1 >= static_cast<int>(LaunchedExecutorPID.size()))
293+
return -1;
294+
return LaunchedExecutorPID.at(n - 1);
295+
}
296+
#endif

clang/unittests/Interpreter/CMakeLists.txt

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,22 @@
1-
add_distinct_clang_unittest(ClangReplInterpreterTests
1+
set(CLANG_REPL_TEST_SOURCES
22
IncrementalCompilerBuilderTest.cpp
33
IncrementalProcessingTest.cpp
44
InterpreterTest.cpp
55
InterpreterExtensionsTest.cpp
66
CodeCompletionTest.cpp
7+
)
78

9+
if(TARGET compiler-rt)
10+
list(APPEND CLANG_REPL_TEST_SOURCES
11+
OutOfProcessInterpreterTests.cpp
12+
)
13+
message(STATUS "Compiler-RT found, enabling out of process JIT tests")
14+
endif()
15+
16+
add_distinct_clang_unittest(ClangReplInterpreterTests
17+
${CLANG_REPL_TEST_SOURCES}
18+
19+
PARTIAL_SOURCES_INTENDED
820
EXPORT_SYMBOLS
921

1022
CLANG_LIBS
@@ -26,6 +38,14 @@ add_distinct_clang_unittest(ClangReplInterpreterTests
2638
TargetParser
2739
)
2840

41+
if(TARGET compiler-rt)
42+
add_dependencies(ClangReplInterpreterTests
43+
llvm-jitlink-executor
44+
compiler-rt
45+
)
46+
message(STATUS "Adding dependency on compiler-rt for out of process JIT tests")
47+
endif()
48+
2949
# Exceptions on Windows are not yet supported.
3050
if(NOT WIN32)
3151
add_subdirectory(ExceptionTests)

clang/unittests/Interpreter/InterpreterTest.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
#include "clang/AST/Decl.h"
1616
#include "clang/AST/DeclGroup.h"
1717
#include "clang/AST/Mangle.h"
18+
#include "clang/Basic/Version.h"
19+
#include "clang/Config/config.h"
1820
#include "clang/Frontend/CompilerInstance.h"
1921
#include "clang/Frontend/TextDiagnosticPrinter.h"
2022
#include "clang/Interpreter/Interpreter.h"
23+
#include "clang/Interpreter/RemoteJITUtils.h"
2124
#include "clang/Interpreter/Value.h"
2225
#include "clang/Sema/Lookup.h"
2326
#include "clang/Sema/Sema.h"
27+
#include "llvm/Support/Error.h"
28+
#include "llvm/TargetParser/Host.h"
2429

2530
#include "llvm/TargetParser/Host.h"
2631

@@ -34,6 +39,12 @@ int Global = 42;
3439
REPL_EXTERNAL_VISIBILITY int getGlobal() { return Global; }
3540
REPL_EXTERNAL_VISIBILITY void setGlobal(int val) { Global = val; }
3641

42+
#ifdef _WIN32
43+
#define STDIN_FILENO 0
44+
#define STDOUT_FILENO 1
45+
#define STDERR_FILENO 2
46+
#endif
47+
3748
namespace {
3849

3950
class InterpreterTest : public InterpreterTestBase {
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
//===- unittests/Interpreter/OutOfProcessInterpreterTest.cpp --- Interpreter
2+
// tests when Out-of-Process ----===//
3+
//
4+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
// See https://llvm.org/LICENSE.txt for license information.
6+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
//
8+
//===----------------------------------------------------------------------===//
9+
//
10+
// Unit tests for Clang's Interpreter library.
11+
//
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "InterpreterTestFixture.h"
15+
16+
#include "clang/AST/Decl.h"
17+
#include "clang/AST/DeclGroup.h"
18+
#include "clang/AST/Mangle.h"
19+
#include "clang/Basic/Version.h"
20+
#include "clang/Config/config.h"
21+
#include "clang/Frontend/CompilerInstance.h"
22+
#include "clang/Frontend/TextDiagnosticPrinter.h"
23+
#include "clang/Interpreter/Interpreter.h"
24+
#include "clang/Interpreter/RemoteJITUtils.h"
25+
#include "clang/Interpreter/Value.h"
26+
#include "clang/Sema/Lookup.h"
27+
#include "clang/Sema/Sema.h"
28+
#include "llvm/Support/Error.h"
29+
#include "llvm/TargetParser/Host.h"
30+
31+
#include "llvm/TargetParser/Host.h"
32+
33+
#include "gmock/gmock.h"
34+
#include "gtest/gtest.h"
35+
36+
using namespace clang;
37+
38+
llvm::ExitOnError ExitOnError;
39+
40+
#ifdef _WIN32
41+
#define STDIN_FILENO 0
42+
#define STDOUT_FILENO 1
43+
#define STDERR_FILENO 2
44+
#endif
45+
46+
namespace {
47+
48+
using Args = std::vector<const char *>;
49+
50+
static void removePathComponent(unsigned N, llvm::SmallString<256> &Path) {
51+
for (unsigned i = 0; i < N; ++i)
52+
llvm::sys::path::remove_filename(Path);
53+
}
54+
55+
static std::string getExecutorPath() {
56+
llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable(
57+
nullptr, reinterpret_cast<void *>(&getExecutorPath)));
58+
removePathComponent(5, ExecutorPath);
59+
llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor");
60+
return ExecutorPath.str().str();
61+
}
62+
63+
static std::string getOrcRuntimePath() {
64+
llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable(
65+
nullptr, reinterpret_cast<void *>(&getOrcRuntimePath)));
66+
removePathComponent(5, RuntimePath);
67+
llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang",
68+
CLANG_VERSION_MAJOR_STRING, "lib");
69+
70+
llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
71+
if (SystemTriple.isOSBinFormatMachO()) {
72+
llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a");
73+
} else if (SystemTriple.isOSBinFormatELF()) {
74+
llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu",
75+
"liborc_rt.a");
76+
}
77+
78+
return RuntimePath.str().str();
79+
}
80+
81+
static std::unique_ptr<Interpreter>
82+
createInterpreterWithRemoteExecution(const Args &ExtraArgs = {},
83+
DiagnosticConsumer *Client = nullptr) {
84+
Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
85+
llvm::append_range(ClangArgs, ExtraArgs);
86+
auto CB = clang::IncrementalCompilerBuilder();
87+
CB.SetCompilerArgs(ClangArgs);
88+
auto CI = cantFail(CB.CreateCpp());
89+
if (Client)
90+
CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false);
91+
92+
std::unique_ptr<llvm::orc::LLJITBuilder> JB;
93+
94+
llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
95+
96+
if ((SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO())) {
97+
std::string OOPExecutor = getExecutorPath();
98+
std::string OrcRuntimePath = getOrcRuntimePath();
99+
bool UseSharedMemory = false;
100+
std::string SlabAllocateSizeString = "";
101+
std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC;
102+
EPC = ExitOnError(launchExecutor(OOPExecutor, UseSharedMemory,
103+
SlabAllocateSizeString,
104+
[=] { // Lambda defined inline
105+
auto redirect = [](int from, int to) {
106+
if (from != to) {
107+
dup2(from, to);
108+
close(from);
109+
}
110+
};
111+
112+
redirect(0, STDIN_FILENO);
113+
redirect(1, STDOUT_FILENO);
114+
redirect(2, STDERR_FILENO);
115+
116+
setvbuf(stdout, nullptr, _IONBF, 0);
117+
setvbuf(stderr, nullptr, _IONBF, 0);
118+
}));
119+
if (EPC) {
120+
CB.SetTargetTriple(EPC->getTargetTriple().getTriple());
121+
JB = ExitOnError(clang::Interpreter::createLLJITBuilder(std::move(EPC),
122+
OrcRuntimePath));
123+
}
124+
}
125+
126+
return cantFail(clang::Interpreter::create(std::move(CI), std::move(JB)));
127+
}
128+
129+
static size_t DeclsSize(TranslationUnitDecl *PTUDecl) {
130+
return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end());
131+
}
132+
133+
TEST_F(InterpreterTestBase, SanityWithRemoteExecution) {
134+
if (!HostSupportsJIT())
135+
GTEST_SKIP();
136+
137+
std::unique_ptr<Interpreter> Interp = createInterpreterWithRemoteExecution();
138+
139+
using PTU = PartialTranslationUnit;
140+
141+
PTU &R1(cantFail(Interp->Parse("void g(); void g() {}")));
142+
EXPECT_EQ(2U, DeclsSize(R1.TUPart));
143+
144+
PTU &R2(cantFail(Interp->Parse("int i;")));
145+
EXPECT_EQ(1U, DeclsSize(R2.TUPart));
146+
}
147+
148+
} // end anonymous namespace

0 commit comments

Comments
 (0)