From 98dd7074ab775c6fe362e60aa257ffa4ce98edb7 Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Tue, 11 Dec 2018 11:23:11 -0800 Subject: [PATCH 1/8] Runtime Provisioner Interface --- .../glow/Runtime/Provisioner/Provisioner.h | 48 +++++++++++++++++++ lib/Runtime/Provisioner/Provisioner.cpp | 32 +++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 include/glow/Runtime/Provisioner/Provisioner.h create mode 100644 lib/Runtime/Provisioner/Provisioner.cpp diff --git a/include/glow/Runtime/Provisioner/Provisioner.h b/include/glow/Runtime/Provisioner/Provisioner.h new file mode 100644 index 0000000000..f5d72dc425 --- /dev/null +++ b/include/glow/Runtime/Provisioner/Provisioner.h @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GLOW_RUNTIME_PROVISIONER_H +#define GLOW_RUNTIME_PROVISIONER_H + +#include + +namespace glow { +enum ResultCode { + READY, + EXECUTED, + FAILED, + CANCELLED +}; // This will be defined in one common runtime place. +class executionDAG; +class dependencyDAG; +class DeviceManager; +/// The Provisioner is responsible for assigning networks to an actual device. +/// It is a stateless class, relying on information being passed in by the +/// caller. +class Provisioner final { +public: + Provisioner(); + ~Provisioner(); + /// Walks \p networks and assigns each module to a DeviceManager in \p + /// devices. The Provisioner calls the addNetwork method for each + /// DeviceManager and uses the returned networkID to populate \p runDAG. + /// Returns a ResultCode indicating if the operation was a success. + ResultCode provision(dependencyDAG &networks, executionDAG &runDAG, + std::unordered_map &devices); +}; + +} // namespace glow + +#endif // GLOW_RUNTIME_PROVISIONER_H diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp new file mode 100644 index 0000000000..dd5392fd62 --- /dev/null +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "glow/Runtime/Provisioner/Provisioner.h" +#include "glow/Partitioner/Partitioner.h" +#include "glow/Runtime/Executor/Executor.h" + +using namespace glow; +Provisioner::Provisioner() = default; + +ResultCode +Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, + std::unordered_map &devices){ + // Check that there is available space for provisioning. + // Assuming there is space, start provisioning. + // Walk the list of modules and call addNetwork. + // If a module fails to provision try on a different device if available. + +}; \ No newline at end of file From d241bd1cfaa1b841de3abc5e1c47683989de3039 Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Tue, 11 Dec 2018 11:29:13 -0800 Subject: [PATCH 2/8] mend --- lib/Runtime/Provisioner/Provisioner.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index dd5392fd62..eecba2d48d 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -25,8 +25,11 @@ ResultCode Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, std::unordered_map &devices){ // Check that there is available space for provisioning. + // This can be the planning phase, we can start with a greedy approach for + // now building a mapping of deviceID:moduleID. // Assuming there is space, start provisioning. // Walk the list of modules and call addNetwork. + // On success add the deviceID to the executionDAG // If a module fails to provision try on a different device if available. }; \ No newline at end of file From fcfd3e92d567f76396c720e7d99537686e531bab Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Wed, 12 Dec 2018 08:31:59 -0800 Subject: [PATCH 3/8] Work on implementation --- .../glow/Runtime/Provisioner/Provisioner.h | 13 ++++------- lib/Runtime/Provisioner/Provisioner.cpp | 23 ++++++++++--------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/include/glow/Runtime/Provisioner/Provisioner.h b/include/glow/Runtime/Provisioner/Provisioner.h index f5d72dc425..81bfd81b8c 100644 --- a/include/glow/Runtime/Provisioner/Provisioner.h +++ b/include/glow/Runtime/Provisioner/Provisioner.h @@ -16,17 +16,12 @@ #ifndef GLOW_RUNTIME_PROVISIONER_H #define GLOW_RUNTIME_PROVISIONER_H +#include "glow/Runtime/RuntimeTypes.h" #include namespace glow { -enum ResultCode { - READY, - EXECUTED, - FAILED, - CANCELLED -}; // This will be defined in one common runtime place. -class executionDAG; -class dependencyDAG; +namespace runtime { + class DeviceManager; /// The Provisioner is responsible for assigning networks to an actual device. /// It is a stateless class, relying on information being passed in by the @@ -42,7 +37,7 @@ class Provisioner final { ResultCode provision(dependencyDAG &networks, executionDAG &runDAG, std::unordered_map &devices); }; - +} // namespace runtime } // namespace glow #endif // GLOW_RUNTIME_PROVISIONER_H diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index eecba2d48d..f580bcafee 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -15,21 +15,22 @@ */ #include "glow/Runtime/Provisioner/Provisioner.h" -#include "glow/Partitioner/Partitioner.h" -#include "glow/Runtime/Executor/Executor.h" using namespace glow; +using namespace runtime; Provisioner::Provisioner() = default; ResultCode Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, - std::unordered_map &devices){ - // Check that there is available space for provisioning. - // This can be the planning phase, we can start with a greedy approach for - // now building a mapping of deviceID:moduleID. - // Assuming there is space, start provisioning. - // Walk the list of modules and call addNetwork. - // On success add the deviceID to the executionDAG - // If a module fails to provision try on a different device if available. - + std::unordered_map &devices) { + // Check that there is available space for provisioning. + // This can be the planning phase, we can start with a greedy approach for + // now building a mapping of deviceID:moduleID. + for (auto module : dependencyDAG) { + // look for space + } + // Assuming there is space, start provisioning. + // Walk the list of modules and call addNetwork. + // On success add the deviceID to the executionDAG + // If a module fails to provision try on a different device if available. }; \ No newline at end of file From c33ccc40cdb3e59d96ce3a5f9bd7a950201a251e Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Thu, 13 Dec 2018 08:49:26 -0800 Subject: [PATCH 4/8] work on provisioner --- lib/Runtime/Provisioner/Provisioner.cpp | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index f580bcafee..ac0c7400c1 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -15,6 +15,9 @@ */ #include "glow/Runtime/Provisioner/Provisioner.h" +#include "glow/Graph/Graph.h" + +#include using namespace glow; using namespace runtime; @@ -24,13 +27,28 @@ ResultCode Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, std::unordered_map &devices) { // Check that there is available space for provisioning. - // This can be the planning phase, we can start with a greedy approach for - // now building a mapping of deviceID:moduleID. - for (auto module : dependencyDAG) { - // look for space + // This will be planning phase, for the first pass we will just assign in + // order. Later we will want to check if networks are already loaded. + std::unordered_map deviceAssignment; + std::list devicesList; + for (auto deviceID : devices) { + deviceList.push_back(deviceID.first); + } + // Assuming number of devices > number of modules. + for (auto module : networks.modules) { + auto device = deviceList.pop_front(); + deviceAssignment.emplace(device, module); } // Assuming there is space, start provisioning. // Walk the list of modules and call addNetwork. // On success add the deviceID to the executionDAG // If a module fails to provision try on a different device if available. + ////// If assignment fails, how do we get the unique_ptr back? + /// through callback? + for (auto assignment : deviceAssignment) { + deviceID = assignment.first; + module = assignment.second; + device = devices[deviceID]; + device.addNetwork() + } }; \ No newline at end of file From 67d57b84068bf82f16bf69551ef4224b6050e382 Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Tue, 18 Dec 2018 10:33:45 -0800 Subject: [PATCH 5/8] Adding logic for addNetwork on DM --- lib/Runtime/Provisioner/Provisioner.cpp | 52 +++++++++++++++---------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index ac0c7400c1..f2b92da97c 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -17,38 +17,50 @@ #include "glow/Runtime/Provisioner/Provisioner.h" #include "glow/Graph/Graph.h" -#include +#include +#include using namespace glow; using namespace runtime; +using DeviceID = unsigned int; Provisioner::Provisioner() = default; ResultCode Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, - std::unordered_map &devices) { + std::unordered_map &devices) { // Check that there is available space for provisioning. - // This will be planning phase, for the first pass we will just assign in + // This will be the planning phase, for the first pass we will just assign in // order. Later we will want to check if networks are already loaded. - std::unordered_map deviceAssignment; - std::list devicesList; - for (auto deviceID : devices) { - deviceList.push_back(deviceID.first); - } + std::unordered_map deviceAssignment; // Assuming number of devices > number of modules. - for (auto module : networks.modules) { - auto device = deviceList.pop_front(); - deviceAssignment.emplace(device, module); + if (networks.modules.size() > devices.size()) { + return FAILED; + } + // Walk devices and networks.modules and pair them as assignments. + for (auto it_device = devices.begin(), it_module = networks.modules.begin(); + it_device != devices.end() || it_module != networks.modules.end();) { + // Pair Module and Device + deviceAssignment.emplace(it_device->first, std::move(*it_module)); + ++it_module; + ++it_device; } - // Assuming there is space, start provisioning. - // Walk the list of modules and call addNetwork. - // On success add the deviceID to the executionDAG - // If a module fails to provision try on a different device if available. - ////// If assignment fails, how do we get the unique_ptr back? - /// through callback? + // For each assignment: + // Check that the device has space, if not fail. + // Call addNetwork and pass in callback, on success add the deviceID to the + // executionDAG If a module fails to provision return failure, otherwise wait + // until all modules are added then return success. + std::mutex provisioned; + std::unordered_map networkIDs; + std::promise done; + auto ready = done.get_future(); for (auto assignment : deviceAssignment) { - deviceID = assignment.first; - module = assignment.second; + deviceID = assignment->first; + modulePtr = assignment->second.get(); device = devices[deviceID]; - device.addNetwork() + device.addNetwork(std::move(assignment.second), + [&provisioned, &networkIDs, modulePtr, &done]() { + provisioned.lock(); + provisioned.unlock(); + }) } }; \ No newline at end of file From 337f5f501a73a6dbbb8d4c61f5c555d9c37f484f Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Thu, 20 Dec 2018 13:26:51 -0800 Subject: [PATCH 6/8] added callback --- lib/Runtime/Provisioner/Provisioner.cpp | 75 ++++++++++++++++++------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index f2b92da97c..050cfcd384 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -25,42 +25,79 @@ using namespace runtime; using DeviceID = unsigned int; Provisioner::Provisioner() = default; +void addNetworkCallback(std::promise &promise, NetworkIDty id, + ResultCode result, std::mutex &cbMutex, + std::unordered_map &networkIDs, + Module *moduleID, unsigned int networkCount) { + cbMutex.lock(); + if (result == READY) { + networkIDs.emplace(moduleID, id); + if (networkIDs.size() == networkCount) { + promise.set_value(true); + } + } else { + promise.set_value(false); + } + cbMutex.unlock(); +} + ResultCode Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, std::unordered_map &devices) { // Check that there is available space for provisioning. // This will be the planning phase, for the first pass we will just assign in // order. Later we will want to check if networks are already loaded. - std::unordered_map deviceAssignment; + std::unordered_map deviceAssignment; // Assuming number of devices > number of modules. if (networks.modules.size() > devices.size()) { return FAILED; } // Walk devices and networks.modules and pair them as assignments. - for (auto it_device = devices.begin(), it_module = networks.modules.begin(); - it_device != devices.end() || it_module != networks.modules.end();) { + for (auto itDevice = devices.begin(), itModule = networks.modules.begin(); + itDevice != devices.end() || itModule != networks.modules.end();) { // Pair Module and Device - deviceAssignment.emplace(it_device->first, std::move(*it_module)); - ++it_module; - ++it_device; + deviceAssignment.emplace(std::move(*itModule), itDevice->first, ); + ++itModule; + ++itDevice; } // For each assignment: // Check that the device has space, if not fail. - // Call addNetwork and pass in callback, on success add the deviceID to the - // executionDAG If a module fails to provision return failure, otherwise wait + // Call addNetwork and pass in callback, on success add the network ID to + // networkIDs. If a module fails to provision return failure, otherwise wait // until all modules are added then return success. - std::mutex provisioned; - std::unordered_map networkIDs; - std::promise done; - auto ready = done.get_future(); + std::mutex addNetworkMutex; + std::unordered_map networkIDs; + std::promise addNetwork; + unsigned int networkCount = deviceAssignment.size(); + auto ready = addNetwork.get_future(); for (auto assignment : deviceAssignment) { - deviceID = assignment->first; - modulePtr = assignment->second.get(); + deviceID = assignment->second; + auto modulePtr = assignment->first.get(); device = devices[deviceID]; - device.addNetwork(std::move(assignment.second), - [&provisioned, &networkIDs, modulePtr, &done]() { - provisioned.lock(); - provisioned.unlock(); - }) + auto networkID = device.getNextDeviceNetworkID(); + device.addNetwork(networkID, std::move(assignment.second), + [&addNetworkMutex, &networkIDs, modulePtr, &addNetwork, + networkCount](NetworkIDty id, ResultCode result) { + addNetworkCallback(addNetwork, id, result, + addNetworkMutex, networkIDs, + modulePtr, networkCount); + }); + } + auto result = ready.get(); + if (!ready) { + return FAILED; + } else { + // Fill in the executionDAG. + for (auto network : networkIDs) { + auto networkID = network->second; + auto moduleID = network->first; + runDAG.networks.push_back(networkID); + runDAG.devices.emplace(networkID, deviceAssignment[moduleID]); + std::vector dependencies; + std::vector dependents; + } + for (auto root : networks.roots) { + runDAG.roots.push_back(networkIDs[root]); + } } }; \ No newline at end of file From f770ac6404823618934c1fbcc03b979112f22d33 Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Mon, 7 Jan 2019 16:16:23 -0800 Subject: [PATCH 7/8] work on provisioner and backends to support compile without collecting constants --- include/glow/Backends/Backend.h | 6 +- include/glow/Backends/BackendUtils.h | 4 +- include/glow/Backends/CompiledFunction.h | 5 ++ .../glow/Runtime/Provisioner/Provisioner.h | 65 ++++++++++++++- lib/Backends/BackendUtils.cpp | 13 +-- lib/Backends/CPU/CPUBackend.cpp | 13 +-- lib/Backends/CPU/CPUBackend.h | 6 +- lib/Backends/Interpreter/Interpreter.cpp | 6 +- lib/Backends/Interpreter/Interpreter.h | 6 +- lib/Backends/OpenCL/OpenCL.cpp | 6 +- lib/Backends/OpenCL/OpenCL.h | 6 +- lib/Runtime/Provisioner/Provisioner.cpp | 80 ++++++++++++++----- 12 files changed, 170 insertions(+), 46 deletions(-) diff --git a/include/glow/Backends/Backend.h b/include/glow/Backends/Backend.h index 8750aced84..0325cb6c73 100644 --- a/include/glow/Backends/Backend.h +++ b/include/glow/Backends/Backend.h @@ -41,7 +41,8 @@ class Backend { virtual ~Backend() = default; /// Generate code for input function \param F. - virtual std::unique_ptr compile(Function *F) const = 0; + virtual std::unique_ptr + compile(Function *F, bool collectConstants = true) const = 0; /// Save the bundle for \p F for a later standalone execution /// in \p outputDir. Make \p networkName the function name for @@ -91,7 +92,8 @@ class BackendUsingGlowIR : public Backend { /// maps the graph to the concrete execution environment for a specific /// function. This is used only for unit testing. virtual std::unique_ptr - compileIR(std::unique_ptr IR) const = 0; + compileIR(std::unique_ptr IR, + bool collectConstants = true) const = 0; }; } // namespace glow diff --git a/include/glow/Backends/BackendUtils.h b/include/glow/Backends/BackendUtils.h index 20ff23eef9..055250730a 100644 --- a/include/glow/Backends/BackendUtils.h +++ b/include/glow/Backends/BackendUtils.h @@ -16,7 +16,6 @@ #ifndef GLOW_BACKENDS_BACKENDUTILS_H #define GLOW_BACKENDS_BACKENDUTILS_H -#include "glow/Backends/CompiledFunction.h" #include "glow/CodeGen/MemoryAllocator.h" #include "glow/IR/IR.h" @@ -79,7 +78,8 @@ class RuntimeBundle { runtime::RuntimeBundle generateRuntimeBundle(const IRFunction &F, MemoryAllocator &constantAllocator, MemoryAllocator &placeholderAllocator, - MemoryAllocator &activationsAllocator); + MemoryAllocator &activationsAllocator, + bool collectConstants = true); } // end namespace glow #endif // GLOW_BACKENDS_BACKENDUTILS_H diff --git a/include/glow/Backends/CompiledFunction.h b/include/glow/Backends/CompiledFunction.h index c80fdf614f..90e3e1d95d 100644 --- a/include/glow/Backends/CompiledFunction.h +++ b/include/glow/Backends/CompiledFunction.h @@ -16,12 +16,15 @@ #ifndef GLOW_BACKENDS_COMPILEDFUNCTION_H #define GLOW_BACKENDS_COMPILEDFUNCTION_H +#include "glow/Backends/BackendUtils.h" #include "glow/Graph/Nodes.h" + #include namespace glow { class Context; + /// Interface for executing a compiled function. class CompiledFunction { public: @@ -41,6 +44,8 @@ class CompiledFunction { virtual void afterRun(const Context &ctx) = 0; /// Final cleanup. Release memory, reset device. virtual void tearDownRuns() = 0; + /// Return RuntimeBundle containing symbol information. + virtual runtime::RuntimeBundle getRuntimeBundle(); }; } // end namespace glow diff --git a/include/glow/Runtime/Provisioner/Provisioner.h b/include/glow/Runtime/Provisioner/Provisioner.h index 81bfd81b8c..5e1aa47868 100644 --- a/include/glow/Runtime/Provisioner/Provisioner.h +++ b/include/glow/Runtime/Provisioner/Provisioner.h @@ -16,8 +16,63 @@ #ifndef GLOW_RUNTIME_PROVISIONER_H #define GLOW_RUNTIME_PROVISIONER_H -#include "glow/Runtime/RuntimeTypes.h" +// #include "glow/Runtime/RuntimeTypes.h" + +////////////////////////////////////////////////////////// +#include "glow/Backends/Backend.h" +#include "glow/Backends/BackendUtils.h" +#include "glow/Backends/CompiledFunction.h" +#include "glow/Graph/Graph.h" + +#include #include +#include + +namespace glow { +namespace runtime { + +using NetworkIDty = size_t; +using DeviceIDty = size_t; + +/// Enum to communicate results when communicating with device at initialization +/// and runtime. +enum ResultCode { READY, EXECUTED, FAILED, CANCELLED }; + +/// Data structure that contains device constraint information for each device. +/// Used to communicate memory constraints and later costs to the Partitioner. +struct DeviceInfo { + /// Available memory on device in bytes. + uint64_t availableMemory; +}; + +/// Individual Node in the DAG for a given network. This contains all the +/// information needed to run the sub-network at inference time. +struct DAGNode { + /// The children of this node, these are nodes that depend on the current + /// node. + std::vector children; + /// Pointers to the parents of this node. This is used by the executor for + /// determining if a given node has all dependencies met. + std::vector parents; + /// ID of the deviceManager that this network is assigned to. + DeviceIDty deviceID; + /// The logicalDevice is an output of the Partitioner to indicate that two + /// networks should be assigned to the same device. + DeviceIDty logicalDevice; + /// Name assigned to the sub-network, this is the id that will be passed to + /// the DeviceManager when requesting a run of the network. + std::string name; + /// Runtime bundle containing all the symbol information for this network at + /// runtime. + RuntimeBundle runtimeBundle; +}; + +} // namespace runtime +} // namespace glow +/////////////////////////////////////////////// + +#include +#include namespace glow { namespace runtime { @@ -34,8 +89,12 @@ class Provisioner final { /// devices. The Provisioner calls the addNetwork method for each /// DeviceManager and uses the returned networkID to populate \p runDAG. /// Returns a ResultCode indicating if the operation was a success. - ResultCode provision(dependencyDAG &networks, executionDAG &runDAG, - std::unordered_map &devices); + ResultCode provision(std::vector &networks, + std::unordered_map &devices, + Module &module); + +private: + std::unique_ptr backend_; }; } // namespace runtime } // namespace glow diff --git a/lib/Backends/BackendUtils.cpp b/lib/Backends/BackendUtils.cpp index ec1d2eb23c..2840fb4bab 100644 --- a/lib/Backends/BackendUtils.cpp +++ b/lib/Backends/BackendUtils.cpp @@ -55,11 +55,10 @@ runtime::RuntimeBundle::getSymbolInfo(const Named *v) const { return it->second; } -runtime::RuntimeBundle -glow::generateRuntimeBundle(const IRFunction &F, - MemoryAllocator &constantAllocator, - MemoryAllocator &placeholderAllocator, - MemoryAllocator &activationsAllocator) { +runtime::RuntimeBundle glow::generateRuntimeBundle( + const IRFunction &F, MemoryAllocator &constantAllocator, + MemoryAllocator &placeholderAllocator, + MemoryAllocator &activationsAllocator, bool collectConstants) { // Handle Constants, Placeholders, and Activations, in that order. // Symbol table mapping symbol name to offset for runtime. std::unordered_map symbolTable; @@ -143,6 +142,8 @@ glow::generateRuntimeBundle(const IRFunction &F, runtime::RuntimeBundle info(symbolTable, constantMaxSize, placeholderMaxSize, activationsMaxSize); - info.collectConstants(&F); + if (collectConstants) { + info.collectConstants(&F); + } return info; } diff --git a/lib/Backends/CPU/CPUBackend.cpp b/lib/Backends/CPU/CPUBackend.cpp index 84b68b2a9d..bfe107fd0e 100644 --- a/lib/Backends/CPU/CPUBackend.cpp +++ b/lib/Backends/CPU/CPUBackend.cpp @@ -98,7 +98,8 @@ CPUBackend::createIRGen(IRFunction *IR, } std::unique_ptr -CPUBackend::compileIR(std::unique_ptr IR) const { +CPUBackend::compileIR(std::unique_ptr IR, + bool collectConstants) const { AllocationsInfo allocationsInfo; std::unique_ptr irgen = createIRGen(IR.get(), allocationsInfo); irgen->initTargetMachine(target.empty() ? "" : target.getValue(), @@ -118,14 +119,16 @@ CPUBackend::compileIR(std::unique_ptr IR) const { MemoryAllocator constantAllocator("ConstantWeights", 0); MemoryAllocator placeholderAllocator("Placeholders", 0); MemoryAllocator activationsAllocator("Activations", 0); - runtime::RuntimeBundle runtimeInfo = generateRuntimeBundle( - *IR, constantAllocator, placeholderAllocator, activationsAllocator); + runtime::RuntimeBundle runtimeInfo = + generateRuntimeBundle(*IR, constantAllocator, placeholderAllocator, + activationsAllocator, collectConstants); return llvm::make_unique(std::move(JIT), runtimeInfo); } -std::unique_ptr CPUBackend::compile(Function *F) const { +std::unique_ptr +CPUBackend::compile(Function *F, bool collectConstants) const { auto IR = generateAndOptimizeIR(F, shouldShareBuffers()); - return compileIR(std::move(IR)); + return compileIR(std::move(IR), collectConstants); } void CPUBackend::save(Function *F, llvm::StringRef outputDir, diff --git a/lib/Backends/CPU/CPUBackend.h b/lib/Backends/CPU/CPUBackend.h index ebcc765826..caac59ffae 100644 --- a/lib/Backends/CPU/CPUBackend.h +++ b/lib/Backends/CPU/CPUBackend.h @@ -43,9 +43,11 @@ class CPUBackend : public BackendUsingGlowIR { ~CPUBackend() override = default; std::unique_ptr - compileIR(std::unique_ptr IR) const override; + compileIR(std::unique_ptr IR, + bool collectConstants = true) const override; - std::unique_ptr compile(Function *F) const override; + std::unique_ptr + compile(Function *F, bool collectConstants = true) const override; void save(Function *F, llvm::StringRef outputDir, llvm::StringRef networkName) const override; diff --git a/lib/Backends/Interpreter/Interpreter.cpp b/lib/Backends/Interpreter/Interpreter.cpp index ff2676d02c..b18269290a 100644 --- a/lib/Backends/Interpreter/Interpreter.cpp +++ b/lib/Backends/Interpreter/Interpreter.cpp @@ -24,13 +24,15 @@ #include "glow/IR/IR.h" using namespace glow; -std::unique_ptr Interpreter::compile(Function *F) const { +std::unique_ptr +Interpreter::compile(Function *F, bool collectConstants) const { auto IR = generateAndOptimizeIR(F, shouldShareBuffers()); return compileIR(std::move(IR)); } std::unique_ptr -Interpreter::compileIR(std::unique_ptr IR) const { +Interpreter::compileIR(std::unique_ptr IR, + bool collectConstants) const { MemoryAllocator constantWeightsAllocator("ConstantWeights", 0); MemoryAllocator placeholderWeightsAllocator("PlaceholderWeights", 0); MemoryAllocator activationsAllocator("Activations", 0); diff --git a/lib/Backends/Interpreter/Interpreter.h b/lib/Backends/Interpreter/Interpreter.h index 6f1efe8d42..1d6144fbbf 100644 --- a/lib/Backends/Interpreter/Interpreter.h +++ b/lib/Backends/Interpreter/Interpreter.h @@ -35,9 +35,11 @@ class Interpreter final : public BackendUsingGlowIR { ~Interpreter() override = default; std::unique_ptr - compileIR(std::unique_ptr IR) const override; + compileIR(std::unique_ptr IR, + bool collectConstants = true) const override; - std::unique_ptr compile(Function *F) const override; + std::unique_ptr + compile(Function *F, bool collectConstants = true) const override; bool isOpSupported(Kinded::Kind opKind, ElemKind elementTy) const override; diff --git a/lib/Backends/OpenCL/OpenCL.cpp b/lib/Backends/OpenCL/OpenCL.cpp index 1cdff8b5ed..59b4ec9ce4 100644 --- a/lib/Backends/OpenCL/OpenCL.cpp +++ b/lib/Backends/OpenCL/OpenCL.cpp @@ -1514,14 +1514,16 @@ cl_mem OpenCLFunction::allocDeviceBuffer(uint64_t size) { void OpenCLFunction::freeDeviceBuffer(cl_mem buf) { clReleaseMemObject(buf); } std::unique_ptr -OCLBackend::compileIR(std::unique_ptr IR) const { +OCLBackend::compileIR(std::unique_ptr IR, + bool collectConstants) const { MemoryAllocator allocator("GPU", 0xFFFFFFFF); runtime::RuntimeBundle bundle = generateRuntimeBundle(*IR, allocator, allocator, allocator); return llvm::make_unique(std::move(IR), bundle); } -std::unique_ptr OCLBackend::compile(Function *F) const { +std::unique_ptr +OCLBackend::compile(Function *F, bool collectConstants) const { auto IR = generateAndOptimizeIR(F, shouldShareBuffers()); return compileIR(std::move(IR)); } diff --git a/lib/Backends/OpenCL/OpenCL.h b/lib/Backends/OpenCL/OpenCL.h index 3f415b3ca0..0fffa8ceb6 100644 --- a/lib/Backends/OpenCL/OpenCL.h +++ b/lib/Backends/OpenCL/OpenCL.h @@ -162,9 +162,11 @@ class OCLBackend final : public BackendUsingGlowIR { ~OCLBackend() override = default; std::unique_ptr - compileIR(std::unique_ptr IR) const override; + compileIR(std::unique_ptr IR, + bool collectConstants = true) const override; - std::unique_ptr compile(Function *F) const override; + std::unique_ptr + compile(Function *F, bool collectConstants = true) const override; bool transformPostLowering(Function *F, CompilationMode mode) const override; diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index 050cfcd384..58c99476dd 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -15,10 +15,14 @@ */ #include "glow/Runtime/Provisioner/Provisioner.h" +#include "glow/Backends/BackendUtils.h" +#include "glow/Backends/CompiledFunction.h" #include "glow/Graph/Graph.h" #include #include +#include +#include using namespace glow; using namespace runtime; @@ -41,30 +45,70 @@ void addNetworkCallback(std::promise &promise, NetworkIDty id, cbMutex.unlock(); } +void addNodes(std::queue> &nextNodes, + std::vector currentNodes) { + for (int i = 0; i < currentNodes[0]->children.size(); i++) { + std::vector newSet; + for (auto node : currentNodes) { + newSet.push_back(&node->children[i]); + } + nextNodes.push(newSet); + } +} ResultCode -Provisioner::provision(dependencyDAG &networks, executionDAG &runDAG, - std::unordered_map &devices) { - // Check that there is available space for provisioning. - // This will be the planning phase, for the first pass we will just assign in - // order. Later we will want to check if networks are already loaded. - std::unordered_map deviceAssignment; - // Assuming number of devices > number of modules. - if (networks.modules.size() > devices.size()) { - return FAILED; +Provisioner::provision(std::vector &networks, + std::unordered_map &devices, + Module &module) { + // For the first pass we will just assign and load devices in order and update + // the deviceID field of the node. + std::queue> nextNode; + // Process head node, this does not contain a function but serves as an entry + // point for the network. We build a vector of nodes, containing all family + // members of a sub-function. + for (int i; i < networks[0].children.size(); i++) { + std::vector newSet; + for (auto node : networks) { + newSet.push_back(&node.children[i]); + } + nextNode.push(newSet); } - // Walk devices and networks.modules and pair them as assignments. - for (auto itDevice = devices.begin(), itModule = networks.modules.begin(); - itDevice != devices.end() || itModule != networks.modules.end();) { - // Pair Module and Device - deviceAssignment.emplace(std::move(*itModule), itDevice->first, ); - ++itModule; - ++itDevice; + while (!nextNode.empty()) { + std::unordered_map compiledFunctions; + auto nodes = nextNode.front(); + nextNode.pop(); + // Add child nodes to the queue. + addNodes(nextNode, nodes); + // Assign collection of nodes to a device, compile and load the device. + // We will do a round robin assignment of nodes. If there is not space we + // will return an error. + // TODO Add ability to try against another device when currDevice has + // insufficient space. + auto currDevice = devices.begin(); + // Set backend to match the device. + backend_.reset(createBackend(currDevice->second.getBackendKind())); + // Iterate over the nodes, compile them and add them to compiledFunctions. + for (auto node : nodes) { + node->deviceID = currDevice->first; + Function *function = module.getFunction(node->name); + auto compiled = backend_->compile(function, false); + // node->runtimeBundle = compiled->getRuntimeBundle(); //FIXME constants + // in bundle are issues.... + compiledFunctions.emplace(node->name, compiled.get()); + } + // Check if sufficient space on device. + // Load functions on device. + currDevice++; + // Handle wrapping around to start of devices again. + if (currDevice == devices.end()) { + currDevice = devices.begin(); + } } + // For each assignment: // Check that the device has space, if not fail. // Call addNetwork and pass in callback, on success add the network ID to - // networkIDs. If a module fails to provision return failure, otherwise wait - // until all modules are added then return success. + // networkIDs. If a module fails to provision return failure, otherwise + // wait until all modules are added then return success. std::mutex addNetworkMutex; std::unordered_map networkIDs; std::promise addNetwork; From 52de7fd55d68b967738f781d8c8786dec69b5718 Mon Sep 17 00:00:00 2001 From: Garret Catron Date: Mon, 7 Jan 2019 16:45:29 -0800 Subject: [PATCH 8/8] added line ending --- lib/Runtime/Provisioner/Provisioner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Runtime/Provisioner/Provisioner.cpp b/lib/Runtime/Provisioner/Provisioner.cpp index 58c99476dd..b1b863ae82 100644 --- a/lib/Runtime/Provisioner/Provisioner.cpp +++ b/lib/Runtime/Provisioner/Provisioner.cpp @@ -144,4 +144,4 @@ Provisioner::provision(std::vector &networks, runDAG.roots.push_back(networkIDs[root]); } } -}; \ No newline at end of file +};