Skip to content
This repository was archived by the owner on Feb 20, 2023. It is now read-only.

Compilation Manager with Shared Pointer Approach #1402

Closed
Closed
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
8c240b0
add compilation_manager files. Still need to finish
Oct 3, 2020
b355915
made adaptive
Oct 6, 2020
ca14b9d
Merge branch 'master' into compilation-manager
Oct 28, 2020
537457d
Merge branch 'master' into compilation-manager
Oct 30, 2020
0f4e73c
moved over compilation to compilation manager
Nov 2, 2020
288d9db
compilation manager takes in bytecode module instead
Nov 10, 2020
dc447cd
reverting to old version
kjobanputra Nov 19, 2020
aa9bc8a
run without crash but no clean up
wuwenw Nov 25, 2020
5ef3ee4
Merge branch 'master' into compilation-manager
kjobanputra Dec 1, 2020
fcf754c
Merge branch 'compilation-manager' into compilation-manager-wuwenw
kjobanputra Dec 1, 2020
f39ce79
Merge branch 'master' of https://github.com/cmu-db/noisepage into com…
wuwenw Dec 17, 2020
e89eaef
sanctioned_shared_ptr
wuwenw Dec 18, 2020
764a7b2
Merge remote-tracking branch 'upstream/master' into compilation-manag…
kjobanputra Jan 19, 2021
a60d53e
slight changes
kjobanputra Jan 25, 2021
71fda07
fix clang tidy
wuwenw Feb 12, 2021
096310f
Merge branch 'master' of https://github.com/cmu-db/noisepage into com…
wuwenw Feb 12, 2021
17b355f
modify NoisePageServer to make working with relative paths easier, mo…
turingcompl33t Feb 13, 2021
d03db58
handle type system issue that comes up on OLTPBench tests
turingcompl33t Feb 13, 2021
fbd1102
update remaining CI stages with compiled query execution tests
turingcompl33t Feb 13, 2021
224491b
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Feb 16, 2021
a101a88
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Feb 17, 2021
306b7e8
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Feb 19, 2021
cd8026b
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Feb 21, 2021
37b18e1
Merge branch 'master' into ci-compiled-junit-tests
lmwnshn Feb 22, 2021
3c23d9e
address (most) comments from review, still one open question
turingcompl33t Feb 22, 2021
28f3f3b
add some basic tests for testing infrastructure utilities, referring …
turingcompl33t Feb 23, 2021
9a30745
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Feb 26, 2021
a249f0e
Merge branch 'master' into ci-compiled-junit-tests
turingcompl33t Mar 5, 2021
85cd5d6
update timeouts, compiled tests do take longer
turingcompl33t Mar 5, 2021
0bea976
increase timeouts again
turingcompl33t Mar 5, 2021
db23be1
Merge branch 'master' into ci-compiled-junit-tests
mbutrovich Mar 8, 2021
73a7ac2
accept args
wuwenw Mar 9, 2021
a15c26f
accept args
wuwenw Mar 9, 2021
b8ec328
Merge branch 'ci-compiled-junit-tests' of https://github.com/turingco…
wuwenw Mar 16, 2021
81b84f2
Merge remote-tracking branch 'cmudb/master' into compilation-manager-…
wuwenw Apr 5, 2021
903c1a6
change python to run oltp
wuwenw Apr 9, 2021
8866013
format
wuwenw Apr 10, 2021
427fd23
remove debug code
wuwenw Apr 10, 2021
be83349
remove debug code
wuwenw Apr 10, 2021
9a51233
add doc
wuwenw Apr 11, 2021
63c72d3
add doc
wuwenw Apr 11, 2021
d58c487
add doc
wuwenw Apr 11, 2021
5fa583d
add doc
wuwenw Apr 11, 2021
4347d8d
add doc
wuwenw Apr 11, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -369,6 +369,11 @@ pipeline {
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tatp.json --build-type=debug
''', label:'OLTPBench (TATP)'

sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tatp.json --build-type=debug -a bytecode_handlers_path=bytecode_handlers_ir.bc -a execution_mode=1
''', label:'OLTPBench (TATP)'

sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tatp_wal_disabled.json --build-type=debug
2 changes: 1 addition & 1 deletion script/testing/oltpbench/__main__.py
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ def build_server_metadata():
# MUTATE args["server_args"].
server_args = test_json.get("server_args", {})
if server_args:
args["server_args"] = server_args
args["server_args"].update(server_args)

# MUTATE args["continue_on_error"].
args["continue_on_error"] = test_json.get(
39 changes: 39 additions & 0 deletions script/testing/oltpbench/utils.py
Original file line number Diff line number Diff line change
@@ -38,7 +38,46 @@ def parse_command_line_args():
aparser.add_argument("--dry-run",
action='store_true',
help="Start and stop DB server without running the OLTPBench tests")
aparser.add_argument('-a', "--server-arg",
default=[],
action='append',
help="Server commandline arguments.")

args = vars(aparser.parse_args())
args['server_args'] = map_server_args(args.get('server_arg'))
del args['server_arg']

args = {k: v for k, v in args.items() if v}

return args


def map_server_args(server_arg_arr):
"""
Map server arguments from an array of strings into a dictionary.
For example,
Input: ["--arg1=10", "--args2=hello", "--debug"]
Output: {
'--arg1': "10",
'--arg2': "hello",
'--debug': None,
}
Parameters
----------
server_arg_arr : [str]
The server arguments as an array of strings.
Returns
-------
server_arg_map : dict
The server arguments in dictionary form.
"""
server_arg_map = {}
for server_arg in server_arg_arr:
if '=' in server_arg:
key, value = server_arg.split('=', 1)
else:
key, value = server_arg, None
server_arg_map[key] = value
return server_arg_map
2 changes: 2 additions & 0 deletions script/testing/util/test_server.py
Original file line number Diff line number Diff line change
@@ -53,6 +53,8 @@ def __init__(self, args, quiet=False):
db_port = args.get("db_port", constants.DEFAULT_DB_PORT)
build_type = args.get("build_type", "")
server_args = args.get("server_args", {})
append_args = args.get("server_arg", {})
server_args.update(append_args)
db_output_file = args.get("db_output_file", constants.DEFAULT_DB_OUTPUT_FILE)
self.db_instance = NoisePageServer(db_host, db_port, build_type, server_args, db_output_file)

14 changes: 7 additions & 7 deletions src/execution/compiler/executable_query.cpp
Original file line number Diff line number Diff line change
@@ -23,29 +23,29 @@ namespace noisepage::execution::compiler {
//===----------------------------------------------------------------------===//

ExecutableQuery::Fragment::Fragment(std::vector<std::string> &&functions, std::vector<std::string> &&teardown_fn,
std::unique_ptr<vm::Module> module)
common::SanctionedSharedPtr<vm::Module>::Ptr module)
: functions_(std::move(functions)), teardown_fn_(std::move(teardown_fn)), module_(std::move(module)) {}

ExecutableQuery::Fragment::~Fragment() = default;

void ExecutableQuery::Fragment::Run(byte query_state[], vm::ExecutionMode mode) const {
void ExecutableQuery::Fragment::Run(byte query_state[], vm::ExecutionMode mode,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region) const {
using Function = std::function<void(void *)>;

auto exec_ctx = *reinterpret_cast<exec::ExecutionContext **>(query_state);
if (exec_ctx->GetTxn()->MustAbort()) {
return;
}
for (const auto &func_name : functions_) {
Function func;
if (!module_->GetFunction(func_name, mode, &func)) {
if (!module_->GetFunction(func_name, mode, &func, module_, context_region)) {
throw EXECUTION_EXCEPTION(fmt::format("Could not find function '{}' in query fragment.", func_name),
common::ErrorCode::ERRCODE_INTERNAL_ERROR);
}
try {
func(query_state);
} catch (const AbortException &e) {
for (const auto &teardown_name : teardown_fn_) {
if (!module_->GetFunction(teardown_name, mode, &func)) {
if (!module_->GetFunction(teardown_name, mode, &func, module_, context_region)) {
throw EXECUTION_EXCEPTION(fmt::format("Could not find teardown function '{}' in query fragment.", func_name),
common::ErrorCode::ERRCODE_INTERNAL_ERROR);
}
@@ -81,7 +81,7 @@ ExecutableQuery::ExecutableQuery(const planner::AbstractPlanNode &plan, const ex
: plan_(plan),
exec_settings_(exec_settings),
errors_region_(std::make_unique<util::Region>("errors_region")),
context_region_(std::make_unique<util::Region>("context_region")),
context_region_(std::make_shared<util::Region>("context_region")), // NOLINT
errors_(std::make_unique<sema::ErrorReporter>(errors_region_.get())),
ast_context_(std::make_unique<ast::Context>(context_region_.get(), errors_.get())),
query_state_size_(0),
@@ -162,7 +162,7 @@ void ExecutableQuery::Run(common::ManagedPointer<exec::ExecutionContext> exec_ct

// Now run through fragments.
for (const auto &fragment : fragments_) {
fragment->Run(query_state.get(), mode);
fragment->Run(query_state.get(), mode, context_region_);
}

// We do not currently re-use ExecutionContexts. However, this is unset to help ensure
12 changes: 8 additions & 4 deletions src/execution/vm/module.cpp
Original file line number Diff line number Diff line change
@@ -22,7 +22,9 @@ namespace noisepage::execution::vm {
class Module::AsyncCompileTask : public tbb::task {
public:
// Construct an asynchronous compilation task to compile the the module
explicit AsyncCompileTask(Module *module) : module_(module) {}
explicit AsyncCompileTask(common::SanctionedSharedPtr<vm::Module>::Ptr module,
common::SanctionedSharedPtr<util::Region>::Ptr context_region)
: module_(std::move(module)), context_region_(std::move(context_region)) {}

// Execute
tbb::task *execute() override {
@@ -33,7 +35,8 @@ class Module::AsyncCompileTask : public tbb::task {
}

private:
Module *module_;
common::SanctionedSharedPtr<vm::Module>::Ptr module_;
common::SanctionedSharedPtr<util::Region>::Ptr context_region_;
};

// ---------------------------------------------------------
@@ -285,8 +288,9 @@ void Module::CompileToMachineCode() {
});
}

void Module::CompileToMachineCodeAsync() {
auto *compile_task = new (tbb::task::allocate_root()) AsyncCompileTask(this);
void Module::CompileToMachineCodeAsync(const common::SanctionedSharedPtr<vm::Module>::Ptr &module,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region) {
auto *compile_task = new (tbb::task::allocate_root()) AsyncCompileTask(module, context_region);
tbb::task::enqueue(*compile_task);
}

11 changes: 7 additions & 4 deletions src/include/execution/compiler/executable_query.h
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@

#include "common/macros.h"
#include "common/managed_pointer.h"
#include "common/sanctioned_shared_pointer.h"
#include "execution/ast/ast_fwd.h"
#include "execution/exec_defs.h"
#include "execution/vm/vm_defs.h"
@@ -68,7 +69,7 @@ class ExecutableQuery {
* @param module The module that contains the functions.
*/
Fragment(std::vector<std::string> &&functions, std::vector<std::string> &&teardown_fns,
std::unique_ptr<vm::Module> module);
common::SanctionedSharedPtr<vm::Module>::Ptr module);

/**
* Destructor.
@@ -79,8 +80,10 @@ class ExecutableQuery {
* Run this fragment using the provided opaque query state object.
* @param query_state The query state.
* @param mode The execution mode to run the query with.
* @param context_region The sanctionedSharedPtr of context region
*/
void Run(std::byte query_state[], vm::ExecutionMode mode) const;
void Run(std::byte query_state[], vm::ExecutionMode mode,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region) const;

/**
* @return True if this fragment is compiled and executable.
@@ -95,7 +98,7 @@ class ExecutableQuery {
std::vector<std::string> teardown_fn_;

// The module.
std::unique_ptr<vm::Module> module_;
common::SanctionedSharedPtr<vm::Module>::Ptr module_;
};

/**
@@ -167,7 +170,7 @@ class ExecutableQuery {
// The execution settings used for code generation.
const exec::ExecutionSettings &exec_settings_;
std::unique_ptr<util::Region> errors_region_;
std::unique_ptr<util::Region> context_region_;
common::SanctionedSharedPtr<util::Region>::Ptr context_region_;
// The AST error reporter.
std::unique_ptr<sema::ErrorReporter> errors_;
// The AST context used to generate the TPL AST.
2 changes: 2 additions & 0 deletions src/include/execution/vm/bytecode_function_info.h
Original file line number Diff line number Diff line change
@@ -197,6 +197,8 @@ class FunctionInfo {
*/
FunctionInfo(FunctionId id, std::string name, ast::FunctionType *func_type);

~FunctionInfo() = default;

/**
* Allocate a new function parameter.
* @param type The TPL type of the parameter.
23 changes: 19 additions & 4 deletions src/include/execution/vm/module.h
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
#include <string>
#include <utility>

#include "common/sanctioned_shared_pointer.h"
#include "execution/ast/type.h"
#include "execution/vm/bytecode_module.h"
#include "execution/vm/llvm_engine.h"
@@ -41,6 +42,8 @@ class Module {
* @param llvm_module The compiled code.
*/
Module(std::unique_ptr<BytecodeModule> bytecode_module, std::unique_ptr<LLVMEngine::CompiledModule> llvm_module);
// TODO(Wuwen): Delete this line. Only for demo
~Module() = default;

/**
* This class cannot be copied or moved.
@@ -73,10 +76,14 @@ class Module {
* @param name The name of the function the caller wants.
* @param exec_mode The mode of the function that the caller wants.
* @param[out] func The function wrapper we use to wrap the TPL function.
* @param module a sanctionedSharedPtr of module
* @param context_region a sanctionedSharedPtr of context_region
* @return True if the function was found and the output parameter was set.
*/
template <typename Ret, typename... ArgTypes>
bool GetFunction(const std::string &name, ExecutionMode exec_mode, std::function<Ret(ArgTypes...)> *func);
bool GetFunction(const std::string &name, ExecutionMode exec_mode, std::function<Ret(ArgTypes...)> *func,
const common::SanctionedSharedPtr<vm::Module>::Ptr &module = nullptr,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region = nullptr);

/**
* Return the raw function implementation for the function in this module with the given function
@@ -102,6 +109,7 @@ class Module {
private:
friend class VM; // For the VM to access raw bytecode.
friend class test::BytecodeTrampolineTest; // For the tests to check private methods.
friend class CompilationManager; // For handling the JIT compilation of the module.

// This class encapsulates the ability to asynchronously JIT compile a module.
class AsyncCompileTask;
@@ -148,7 +156,11 @@ class Module {

// Compile this module into machine code. This is a non-blocking call that
// triggers a compilation in the background.
void CompileToMachineCodeAsync();
void CompileToMachineCodeAsync(const common::SanctionedSharedPtr<vm::Module>::Ptr &module,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region);

// Gets the flag that indicates if the JIT compilation has occurred.
std::once_flag GetCompiledFlag();

private:
// The module containing all TBC (i.e., bytecode) for the TPL program.
@@ -191,7 +203,9 @@ inline void CopyAll(uint8_t *buffer, const HeadT &head, const RestT &... rest) {

template <typename Ret, typename... ArgTypes>
inline bool Module::GetFunction(const std::string &name, const ExecutionMode exec_mode,
std::function<Ret(ArgTypes...)> *func) {
std::function<Ret(ArgTypes...)> *func,
const common::SanctionedSharedPtr<vm::Module>::Ptr &module,
const common::SanctionedSharedPtr<util::Region>::Ptr &context_region) {
// Lookup function
const FunctionInfo *func_info = bytecode_module_->LookupFuncInfoByName(name);

@@ -208,7 +222,8 @@ inline bool Module::GetFunction(const std::string &name, const ExecutionMode exe

switch (exec_mode) {
case ExecutionMode::Adaptive: {
CompileToMachineCodeAsync();
// TODO(kjobanputra): call into compilation manager here with new module
CompileToMachineCodeAsync(module, context_region);
FALLTHROUGH;
}
case ExecutionMode::Interpret: {
6 changes: 3 additions & 3 deletions src/include/execution/vm/vm_defs.h
Original file line number Diff line number Diff line change
@@ -7,13 +7,13 @@ namespace noisepage::execution::vm {
*/
enum class ExecutionMode : uint8_t {
// Always execute in interpreted mode
Interpret,
Interpret = 0,
// Execute in interpreted mode, but trigger a compilation asynchronously. As
// compiled code becomes available, seamlessly swap it in and execute mixed
// interpreter and compiled code.
Adaptive,
Adaptive = 1,
// Compile and generate all machine code before executing the function
Compiled
Compiled = 2
};

} // namespace noisepage::execution::vm
17 changes: 17 additions & 0 deletions src/include/main/db_main.h
Original file line number Diff line number Diff line change
@@ -476,6 +476,8 @@ class DBMain {
"TrafficCopLayer needs the CatalogLayer.");
NOISEPAGE_ASSERT(use_stats_storage_ && stats_storage != DISABLED, "TrafficCopLayer needs StatsStorage.");
NOISEPAGE_ASSERT(use_execution_ && execution_layer != DISABLED, "TrafficCopLayer needs ExecutionLayer.");
// TODO(Wuwen): a managed pointer of execution layer is passed to the traffic_cop for easy access, but we might
// want a better way
traffic_cop = std::make_unique<trafficcop::TrafficCop>(
txn_layer->GetTransactionManager(), catalog_layer->GetCatalog(),
common::ManagedPointer(replication_manager), common::ManagedPointer(settings_manager),
@@ -991,11 +993,26 @@ class DBMain {
optimizer_timeout_ = static_cast<uint64_t>(settings_manager->GetInt(settings::Param::task_execution_timeout));
use_query_cache_ = settings_manager->GetBool(settings::Param::use_query_cache);

// TODO(Wuwen): It's redundant
execution_mode_ = settings_manager->GetBool(settings::Param::compiled_query_execution)
? execution::vm::ExecutionMode::Compiled
: execution::vm::ExecutionMode::Interpret;
bytecode_handlers_path_ = settings_manager->GetString(settings::Param::bytecode_handlers_path);

switch (static_cast<execution::vm::ExecutionMode>(settings_manager->GetInt(settings::Param::execution_mode))) {
case execution::vm::ExecutionMode::Interpret:
execution_mode_ = execution::vm::ExecutionMode::Interpret;
break;
case execution::vm::ExecutionMode::Adaptive:
execution_mode_ = execution::vm::ExecutionMode::Adaptive;
break;
case execution::vm::ExecutionMode::Compiled:
execution_mode_ = execution::vm::ExecutionMode::Compiled;
break;
default:
break;
}

query_trace_metrics_ = settings_manager->GetBool(settings::Param::query_trace_metrics_enable);
pipeline_metrics_ = settings_manager->GetBool(settings::Param::pipeline_metrics_enable);
pipeline_metrics_sample_rate_ = settings_manager->GetInt(settings::Param::pipeline_metrics_sample_rate);
10 changes: 10 additions & 0 deletions src/include/settings/settings_defs.h
Original file line number Diff line number Diff line change
@@ -474,4 +474,14 @@ SETTING_string(
false,
noisepage::settings::Callbacks::NoOp
)

SETTING_int(
execution_mode,
"The execution mode specified by the user, 0 = Interpret, 1 = Adaptive, 2 = Compiled",
0,
0,
2,
false,
noisepage::settings::Callbacks::NoOp
)
// clang-format on
18 changes: 11 additions & 7 deletions util/execution/tpl.cpp
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
#include <string>
#include <utility>

#include "common/sanctioned_shared_pointer.h"
#include "execution/ast/ast_dump.h"
#include "execution/ast/ast_pretty_print.h"
#include "execution/exec/execution_context.h"
@@ -114,9 +115,11 @@ static void CompileAndRun(const std::string &source, const std::string &name = "

// Let's parse the source
util::Region err_region{"tmp-error-region"};
util::Region context_region{"tmp-context-region"};
common::SanctionedSharedPtr<util::Region>::Ptr context_region =
std::make_shared<util::Region>("tmp-context-region"); // NOLINT

sema::ErrorReporter error_reporter{&err_region};
ast::Context context(&context_region, &error_reporter);
ast::Context context(context_region.get(), &error_reporter);

parsing::Scanner scanner(source.data(), source.length());
parsing::Parser parser(&scanner, &context);
@@ -183,7 +186,8 @@ static void CompileAndRun(const std::string &source, const std::string &name = "
bytecode_module->Dump(std::cout); // NOLINT
}

auto module = std::make_unique<vm::Module>(std::move(bytecode_module));
common::SanctionedSharedPtr<vm::Module>::Ptr module =
std::make_shared<vm::Module>(std::move(bytecode_module)); // NOLINT

//
// Interpret
@@ -195,14 +199,14 @@ static void CompileAndRun(const std::string &source, const std::string &name = "

if (IS_SQL) {
std::function<int32_t(exec::ExecutionContext *)> main;
if (!module->GetFunction("main", vm::ExecutionMode::Interpret, &main)) {
if (!module->GetFunction("main", vm::ExecutionMode::Interpret, &main, module, context_region)) {
EXECUTION_LOG_ERROR("Missing 'main' entry function with signature (*ExecutionContext)->int32");
return;
}
EXECUTION_LOG_INFO("VM main() returned: {}", main(&exec_ctx));
} else {
std::function<int32_t()> main;
if (!module->GetFunction("main", vm::ExecutionMode::Interpret, &main)) {
if (!module->GetFunction("main", vm::ExecutionMode::Interpret, &main, module, context_region)) {
EXECUTION_LOG_ERROR("Missing 'main' entry function with signature ()->int32");
return;
}
@@ -219,14 +223,14 @@ static void CompileAndRun(const std::string &source, const std::string &name = "

if (IS_SQL) {
std::function<int32_t(exec::ExecutionContext *)> main;
if (!module->GetFunction("main", vm::ExecutionMode::Adaptive, &main)) {
if (!module->GetFunction("main", vm::ExecutionMode::Adaptive, &main, module, context_region)) {
EXECUTION_LOG_ERROR("Missing 'main' entry function with signature (*ExecutionContext)->int32");
return;
}
EXECUTION_LOG_INFO("ADAPTIVE main() returned: {}", main(&exec_ctx));
} else {
std::function<int32_t()> main;
if (!module->GetFunction("main", vm::ExecutionMode::Adaptive, &main)) {
if (!module->GetFunction("main", vm::ExecutionMode::Adaptive, &main, module, context_region)) {
EXECUTION_LOG_ERROR("Missing 'main' entry function with signature ()->int32");
return;
}