From d6ea0d56368d348d9035acc311a44626fa96e979 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 17 Aug 2022 09:10:01 -0700 Subject: [PATCH 01/31] feat: Jetson specific workspace file Signed-off-by: Naren Dasan Signed-off-by: Naren Dasan --- py/setup.py | 17 +++-- toolchains/BUILD | 9 +++ toolchains/jp_workspaces/WORKSPACE.jp50 | 96 +++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 toolchains/jp_workspaces/WORKSPACE.jp50 diff --git a/py/setup.py b/py/setup.py index fc7019ce31..88626a9e04 100644 --- a/py/setup.py +++ b/py/setup.py @@ -72,12 +72,18 @@ def get_git_revision_short_hash() -> str: elif version == "4.6": JETPACK_VERSION = "4.6" elif version == "5.0": - JETPACK_VERSION = "4.6" + JETPACK_VERSION = "5.0" + if not JETPACK_VERSION: warnings.warn( - "Assuming jetpack version to be 4.6 or greater, if not use the --jetpack-version option" + "Assuming jetpack version to be 5.0, if not use the --jetpack-version option" + ) + JETPACK_VERSION = "5.0" + + if not CXX11_ABI: + warnings.warn( + "Jetson platform detected but did not see --use-cxx11-abi option, if using a pytorch distribution provided by NVIDIA include this flag" ) - JETPACK_VERSION = "4.6" def which(program): @@ -128,7 +134,10 @@ def build_libtorchtrt_pre_cxx11_abi(develop=True, use_dist_dir=True, cxx11_abi=F print("Jetpack version: 4.5") elif JETPACK_VERSION == "4.6": cmd.append("--platforms=//toolchains:jetpack_4.6") - print("Jetpack version: >=4.6") + print("Jetpack version: 4.6") + elif JETPACK_VERSION == "5.0": + cmd.append("--platforms=//toolchains:jetpack_5.0") + print("Jetpack version: 5.0") if CI_RELEASE: cmd.append("--platforms=//toolchains:ci_rhel_x86_64_linux") diff --git a/toolchains/BUILD b/toolchains/BUILD index 117cdcf498..aa6486d075 100644 --- a/toolchains/BUILD +++ b/toolchains/BUILD @@ -26,6 +26,15 @@ platform( ], ) +platform( + name = "jetpack_5.0", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "@//toolchains/jetpack:4.6", + ], +) + platform( name = "ci_rhel_x86_64_linux", constraint_values = [ diff --git a/toolchains/jp_workspaces/WORKSPACE.jp50 b/toolchains/jp_workspaces/WORKSPACE.jp50 new file mode 100644 index 0000000000..a5c054dd23 --- /dev/null +++ b/toolchains/jp_workspaces/WORKSPACE.jp50 @@ -0,0 +1,96 @@ +workspace(name = "Torch-TensorRT") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz", +) + +load("@rules_python//python:pip.bzl", "pip_install") + +http_archive( + name = "rules_pkg", + sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + +git_repository( + name = "googletest", + commit = "703bd9caab50b139428cea1aaff9974ebee5742e", + remote = "https://github.com/google/googletest", + shallow_since = "1570114335 -0400", +) + +# External dependency for torch_tensorrt if you already have precompiled binaries. +local_repository( + name = "torch_tensorrt", + path = "/opt/conda/lib/python3.8/site-packages/torch_tensorrt", +) + +# CUDA should be installed on the system locally +new_local_repository( + name = "cuda", + build_file = "@//third_party/cuda:BUILD", + path = "/usr/local/cuda-11.4/", +) + +new_local_repository( + name = "cublas", + build_file = "@//third_party/cublas:BUILD", + path = "/usr", +) + +#################################################################################### +# Locally installed dependencies (use in cases of custom dependencies or aarch64) +#################################################################################### + +# NOTE: In the case you are using just the pre-cxx11-abi path or just the cxx11 abi path +# with your local libtorch, just point deps at the same path to satisfy bazel. + +# NOTE: NVIDIA's aarch64 PyTorch (python) wheel file uses the CXX11 ABI unlike PyTorch's standard +# x86_64 python distribution. If using NVIDIA's version just point to the root of the package +# for both versions here and do not use --config=pre-cxx11-abi + +new_local_repository( + name = "libtorch", + path = "/usr/local/lib/python3.8/dist-packages/torch", + build_file = "third_party/libtorch/BUILD" +) + +# NOTE: Unused on aarch64-jetson with NVIDIA provided PyTorch distribu†ion +new_local_repository( + name = "libtorch_pre_cxx11_abi", + path = "/usr/local/lib/python3.8/dist-packages/torch", + build_file = "third_party/libtorch/BUILD" +) + +new_local_repository( + name = "cudnn", + path = "/usr/", + build_file = "@//third_party/cudnn/local:BUILD" +) + +new_local_repository( + name = "tensorrt", + path = "/usr/", + build_file = "@//third_party/tensorrt/local:BUILD" +) + +######################################################################### +# Development Dependencies (optional - comment out on aarch64) +######################################################################### + +pip_install( + name = "devtools_deps", + requirements = "//:requirements-dev.txt", +) From 7f60b18a0496e626fdf57160036df0a79a39efe7 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 17 Aug 2022 09:17:14 -0700 Subject: [PATCH 02/31] docs: Update jetson instructions Signed-off-by: Naren Dasan Signed-off-by: Naren Dasan --- docsrc/tutorials/installation.rst | 70 ++++--------------------------- 1 file changed, 9 insertions(+), 61 deletions(-) diff --git a/docsrc/tutorials/installation.rst b/docsrc/tutorials/installation.rst index ed5dea5df3..937dd1dde9 100644 --- a/docsrc/tutorials/installation.rst +++ b/docsrc/tutorials/installation.rst @@ -303,54 +303,13 @@ Enviorment Setup To build natively on aarch64-linux-gnu platform, configure the ``WORKSPACE`` with local available dependencies. -1. Disable the rules with ``http_archive`` for x86_64 by commenting the following rules: - -.. code-block:: shell - - #http_archive( - # name = "libtorch", - # build_file = "@//third_party/libtorch:BUILD", - # strip_prefix = "libtorch", - # urls = ["https://download.pytorch.org/libtorch/cu102/libtorch-cxx11-abi-shared-with-deps-1.5.1.zip"], - # sha256 = "cf0691493d05062fe3239cf76773bae4c5124f4b039050dbdd291c652af3ab2a" - #) - - #http_archive( - # name = "libtorch_pre_cxx11_abi", - # build_file = "@//third_party/libtorch:BUILD", - # strip_prefix = "libtorch", - # sha256 = "818977576572eadaf62c80434a25afe44dbaa32ebda3a0919e389dcbe74f8656", - # urls = ["https://download.pytorch.org/libtorch/cu102/libtorch-shared-with-deps-1.5.1.zip"], - #) - - # Download these tarballs manually from the NVIDIA website - # Either place them in the distdir directory in third_party and use the --distdir flag - # or modify the urls to "file:////.tar.gz - - #http_archive( - # name = "cudnn", - # urls = ["https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.0.1.13/10.2_20200626/cudnn-10.2-linux-x64-v8.0.1.13.tgz"], - # build_file = "@//third_party/cudnn/archive:BUILD", - # sha256 = "0c106ec84f199a0fbcf1199010166986da732f9b0907768c9ac5ea5b120772db", - # strip_prefix = "cuda" - #) - - #http_archive( - # name = "tensorrt", - # urls = ["https://developer.nvidia.com/compute/machine-learning/tensorrt/secure/7.1/tars/TensorRT-7.1.3.4.Ubuntu-18.04.x86_64-gnu.cuda-10.2.cudnn8.0.tar.gz"], - # build_file = "@//third_party/tensorrt/archive:BUILD", - # sha256 = "9205bed204e2ae7aafd2e01cce0f21309e281e18d5bfd7172ef8541771539d41", - # strip_prefix = "TensorRT-7.1.3.4" - #) - - NOTE: You may also need to configure the CUDA version to 10.2 by setting the path for the cuda new_local_repository - +1. Replace ``WORKSPACE`` with the corresponding WORKSPACE file in ``//toolchains/jp_workspaces`` 2. Configure the correct paths to directory roots containing local dependencies in the ``new_local_repository`` rules: NOTE: If you installed PyTorch using a pip package, the correct path is the path to the root of the python torch package. - In the case that you installed with ``sudo pip install`` this will be ``/usr/local/lib/python3.6/dist-packages/torch``. - In the case you installed with ``pip install --user`` this will be ``$HOME/.local/lib/python3.6/site-packages/torch``. + In the case that you installed with ``sudo pip install`` this will be ``/usr/local/lib/python3.8/dist-packages/torch``. + In the case you installed with ``pip install --user`` this will be ``$HOME/.local/lib/python3.8/site-packages/torch``. In the case you are using NVIDIA compiled pip packages, set the path for both libtorch sources to the same path. This is because unlike PyTorch on x86_64, NVIDIA aarch64 PyTorch uses the CXX11-ABI. If you compiled for source using the pre_cxx11_abi and only would like to @@ -360,27 +319,16 @@ use that library, set the paths to the same path but when you compile make sure new_local_repository( name = "libtorch", - path = "/usr/local/lib/python3.6/dist-packages/torch", + path = "/usr/local/lib/python3.8/dist-packages/torch", build_file = "third_party/libtorch/BUILD" ) new_local_repository( name = "libtorch_pre_cxx11_abi", - path = "/usr/local/lib/python3.6/dist-packages/torch", + path = "/usr/local/lib/python3.8/dist-packages/torch", build_file = "third_party/libtorch/BUILD" ) - new_local_repository( - name = "cudnn", - path = "/usr/", - build_file = "@//third_party/cudnn/local:BUILD" - ) - - new_local_repository( - name = "tensorrt", - path = "/usr/", - build_file = "@//third_party/tensorrt/local:BUILD" - ) Compile C++ Library and Compiler CLI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -389,19 +337,19 @@ Compile C++ Library and Compiler CLI .. code-block:: shell - --platforms //toolchains:jetpack_4.x + --platforms //toolchains:jetpack_x.x Compile Torch-TensorRT library using bazel command: .. code-block:: shell - bazel build //:libtorchtrt --platforms //toolchains:jetpack_4.6 + bazel build //:libtorchtrt --platforms //toolchains:jetpack_5.0 Compile Python API ^^^^^^^^^^^^^^^^^^^^ - NOTE: Due to shifting dependencies locations between Jetpack 4.5 and Jetpack 4.6 there is now a flag for ``setup.py`` which sets the jetpack version (default: 4.6) + NOTE: Due to shifting dependencies locations between Jetpack 4.5 and newer Jetpack verisons there is now a flag for ``setup.py`` which sets the jetpack version (default: 5.0) Compile the Python API using the following command from the ``//py`` directory: @@ -411,4 +359,4 @@ Compile the Python API using the following command from the ``//py`` directory: If you have a build of PyTorch that uses Pre-CXX11 ABI drop the ``--use-cxx11-abi`` flag -If you are building for Jetpack 4.5 add the ``--jetpack-version 4.5`` flag +If you are building for Jetpack 4.5 add the ``--jetpack-version 5.0`` flag From 61adecfcbf4b62c2a4104d9d3103cb19617b1ed1 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 17 Aug 2022 15:42:38 -0700 Subject: [PATCH 03/31] docs: Update docgen task to not require a special container Signed-off-by: Naren Dasan Signed-off-by: Naren Dasan --- .github/workflows/docgen.yml | 2 +- docsrc/WORKSPACE.docs | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docsrc/WORKSPACE.docs diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index cf74831e74..c1d6f0944c 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -47,4 +47,4 @@ jobs: file_pattern: docs/ commit_user_name: Torch-TensorRT Github Bot commit_user_email: torch-tensorrt.github.bot@nvidia.com - commit_author: Torch-TensorRT Github Bot + commit_author: Torch-TensorRT Github Bot \ No newline at end of file diff --git a/docsrc/WORKSPACE.docs b/docsrc/WORKSPACE.docs new file mode 100644 index 0000000000..e69de29bb2 From cf547928fc32d628f0c0f2d7faf0963b5fb5b019 Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Tue, 14 Jun 2022 14:52:45 -0700 Subject: [PATCH 04/31] Checkpoint, initial test and implementation --- .../converters/impl/constant_pad.cpp | 13 +++-------- .../converters/test_constant_pad.cpp | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/core/conversion/converters/impl/constant_pad.cpp b/core/conversion/converters/impl/constant_pad.cpp index 679a23f875..670687cf1d 100644 --- a/core/conversion/converters/impl/constant_pad.cpp +++ b/core/conversion/converters/impl/constant_pad.cpp @@ -21,7 +21,8 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns auto padding = args[1].unwrapToIntList().vec(); int64_t padSize = padding.size(); auto value = args[2].unwrapToScalar().to(); - + at::Tensor value_tensor = torch::tensor(value, util::TRTDataTypeToScalarType(in->getType())); + auto valueTensor = tensor_to_const(ctx, value_tensor); TORCHTRT_CHECK(padSize % 2 == 0, "Length of pad must be even but instead it equals " << padSize); int64_t l_pad = padSize / 2; @@ -55,8 +56,6 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns auto fill_layer = ctx->net->addFill(nvinfer1::Dims{1, {1}}, nvinfer1::FillOperation::kLINSPACE); auto shape_gather_out = ctx->net->addShape(*left_gather_out)->getOutput(0); fill_layer->setInput(0, *shape_gather_out); - at::Tensor value_tensor = torch::tensor(value, torch::kFloat32); - auto valueTensor = tensor_to_const(ctx, value_tensor); fill_layer->setInput(1, *valueTensor); at::Tensor delta_tensor = torch::zeros(inRank); auto deltaTensor = tensor_to_const(ctx, delta_tensor); @@ -69,8 +68,6 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns } else { inDims.d[axis] = padding[padding_index]; auto fill_layer = ctx->net->addFill(inDims, nvinfer1::FillOperation::kLINSPACE); - at::Tensor value_tensor = torch::tensor(value, torch::kFloat32); - auto valueTensor = tensor_to_const(ctx, value_tensor); fill_layer->setInput(1, *valueTensor); at::Tensor delta_tensor = torch::zeros(inRank); auto deltaTensor = tensor_to_const(ctx, delta_tensor); @@ -111,9 +108,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns // fill the right_gather_out with value auto fill_layer = ctx->net->addFill(nvinfer1::Dims{1, {1}}, nvinfer1::FillOperation::kLINSPACE); auto shape_gather_out = ctx->net->addShape(*right_gather_out)->getOutput(0); - fill_layer->setInput(0, *shape_gather_out); - at::Tensor value_tensor = torch::tensor(value, torch::kFloat32); - auto valueTensor = tensor_to_const(ctx, value_tensor); + fill_layer->setInput(0, *shape_gather_out); fill_layer->setInput(1, *valueTensor); at::Tensor delta_tensor = torch::zeros(inRank); auto deltaTensor = tensor_to_const(ctx, delta_tensor); @@ -126,8 +121,6 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns } else { inDims.d[axis] = padding[padding_index + 1]; auto fill_layer = ctx->net->addFill(inDims, nvinfer1::FillOperation::kLINSPACE); - at::Tensor value_tensor = torch::tensor(value, torch::kFloat32); - auto valueTensor = tensor_to_const(ctx, value_tensor); fill_layer->setInput(1, *valueTensor); at::Tensor delta_tensor = torch::zeros(inRank); auto deltaTensor = tensor_to_const(ctx, delta_tensor); diff --git a/tests/core/conversion/converters/test_constant_pad.cpp b/tests/core/conversion/converters/test_constant_pad.cpp index 9b37be4352..c5f0bd8a31 100644 --- a/tests/core/conversion/converters/test_constant_pad.cpp +++ b/tests/core/conversion/converters/test_constant_pad.cpp @@ -28,6 +28,29 @@ TEST(Converters, ATenConstantPad1dTensorConvertsCorrectly) { torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0].reshape_as(jit_results[0]), 2e-6)); } +TEST(Converters, ATenConstantPad1dIntTensorConvertsCorrectly) { + const auto graph = R"IR( + graph(%0 : Tensor): + %1 : int[] = prim::Constant[value=[2, 3]]() + %2 : Scalar = prim::Constant[value=2]() + %3 : Tensor = aten::constant_pad_nd(%0, %1, %2) + return (%3))IR"; + + auto g = std::make_shared(); + torch::jit::parseIR(graph, g.get()); + + auto in1 = at::randint(1, 10, {1, 3, 4}, {at::kCUDA}).toType(at::kInt); + + auto params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {}); + auto jit_results = torch_tensorrt::tests::util::RunGraph(g, params, {in1}); + + params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {}); + auto trt_results = torch_tensorrt::tests::util::RunGraphEngine(g, params, {in1}); + + ASSERT_TRUE( + torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0].reshape_as(jit_results[0]), 2e-6)); +} + TEST(Converters, ATenConstantPad1dRightZeroTensorConvertsCorrectly) { const auto graph = R"IR( graph(%0 : Tensor): From df240790d622b6c2d4d402d870c65fe66eba7396 Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Tue, 14 Jun 2022 17:36:17 -0700 Subject: [PATCH 05/31] Fix error in layer conversion caused by zero/ones tensors of wrong type --- core/conversion/converters/impl/constant_pad.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/conversion/converters/impl/constant_pad.cpp b/core/conversion/converters/impl/constant_pad.cpp index 670687cf1d..da2490eab1 100644 --- a/core/conversion/converters/impl/constant_pad.cpp +++ b/core/conversion/converters/impl/constant_pad.cpp @@ -57,7 +57,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns auto shape_gather_out = ctx->net->addShape(*left_gather_out)->getOutput(0); fill_layer->setInput(0, *shape_gather_out); fill_layer->setInput(1, *valueTensor); - at::Tensor delta_tensor = torch::zeros(inRank); + at::Tensor delta_tensor = torch::zeros(inRank, util::TRTDataTypeToScalarType(in->getType())); auto deltaTensor = tensor_to_const(ctx, delta_tensor); fill_layer->setInput(2, *deltaTensor); auto padTensor = fill_layer->getOutput(0); @@ -69,7 +69,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns inDims.d[axis] = padding[padding_index]; auto fill_layer = ctx->net->addFill(inDims, nvinfer1::FillOperation::kLINSPACE); fill_layer->setInput(1, *valueTensor); - at::Tensor delta_tensor = torch::zeros(inRank); + at::Tensor delta_tensor = torch::zeros(inRank, util::TRTDataTypeToScalarType(in->getType())); auto deltaTensor = tensor_to_const(ctx, delta_tensor); fill_layer->setInput(2, *deltaTensor); auto padTensor = fill_layer->getOutput(0); @@ -110,7 +110,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns auto shape_gather_out = ctx->net->addShape(*right_gather_out)->getOutput(0); fill_layer->setInput(0, *shape_gather_out); fill_layer->setInput(1, *valueTensor); - at::Tensor delta_tensor = torch::zeros(inRank); + at::Tensor delta_tensor = torch::zeros(inRank, util::TRTDataTypeToScalarType(in->getType())); auto deltaTensor = tensor_to_const(ctx, delta_tensor); fill_layer->setInput(2, *deltaTensor); auto padTensor = fill_layer->getOutput(0); @@ -122,7 +122,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns inDims.d[axis] = padding[padding_index + 1]; auto fill_layer = ctx->net->addFill(inDims, nvinfer1::FillOperation::kLINSPACE); fill_layer->setInput(1, *valueTensor); - at::Tensor delta_tensor = torch::zeros(inRank); + at::Tensor delta_tensor = torch::zeros(inRank, util::TRTDataTypeToScalarType(in->getType())); auto deltaTensor = tensor_to_const(ctx, delta_tensor); fill_layer->setInput(2, *deltaTensor); auto padTensor = fill_layer->getOutput(0); From 6aee08451e9e4b776cd6f3cea629d8fd4571ec7d Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Thu, 16 Jun 2022 11:38:36 -0700 Subject: [PATCH 06/31] Sort input values in registerSegmentsOutputs to resolve nondeterminism --- core/partitioning/partitioning.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/partitioning/partitioning.cpp b/core/partitioning/partitioning.cpp index f083f6fbe5..892938d82a 100644 --- a/core/partitioning/partitioning.cpp +++ b/core/partitioning/partitioning.cpp @@ -178,7 +178,8 @@ void resolveTRTNonTensorInputs(PartitionedGraph& segmented_blocks) { void registerSegmentsOutputs(PartitionedGraph& segmented_blocks, torch::jit::Block* block) { // find the corresponding raw values in original global graph for this segmented block's inputs/outputs - std::set input_values; + auto cmp = [](torch::jit::Value* a, torch::jit::Value* b) { return a->unique() < b->unique();}; + std::set input_values(cmp); for (auto& seg_block : segmented_blocks) { for (auto& input : seg_block.raw_inputs()) { input_values.insert(input); From bf6645f0f1422d1833e5de695a6b1ca181a4b030 Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Thu, 18 Aug 2022 10:07:08 -0700 Subject: [PATCH 07/31] lint --- core/partitioning/partitioning.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/partitioning/partitioning.cpp b/core/partitioning/partitioning.cpp index 892938d82a..c329b33ef6 100644 --- a/core/partitioning/partitioning.cpp +++ b/core/partitioning/partitioning.cpp @@ -178,7 +178,7 @@ void resolveTRTNonTensorInputs(PartitionedGraph& segmented_blocks) { void registerSegmentsOutputs(PartitionedGraph& segmented_blocks, torch::jit::Block* block) { // find the corresponding raw values in original global graph for this segmented block's inputs/outputs - auto cmp = [](torch::jit::Value* a, torch::jit::Value* b) { return a->unique() < b->unique();}; + auto cmp = [](torch::jit::Value* a, torch::jit::Value* b) { return a->unique() < b->unique(); }; std::set input_values(cmp); for (auto& seg_block : segmented_blocks) { for (auto& input : seg_block.raw_inputs()) { From 911ab5b9dee788ca1aa668e02e9de8a406e42f82 Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Thu, 18 Aug 2022 10:13:57 -0700 Subject: [PATCH 08/31] lint --- core/conversion/converters/impl/constant_pad.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/conversion/converters/impl/constant_pad.cpp b/core/conversion/converters/impl/constant_pad.cpp index da2490eab1..6d3f1ab609 100644 --- a/core/conversion/converters/impl/constant_pad.cpp +++ b/core/conversion/converters/impl/constant_pad.cpp @@ -108,7 +108,7 @@ auto constant_pad_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns // fill the right_gather_out with value auto fill_layer = ctx->net->addFill(nvinfer1::Dims{1, {1}}, nvinfer1::FillOperation::kLINSPACE); auto shape_gather_out = ctx->net->addShape(*right_gather_out)->getOutput(0); - fill_layer->setInput(0, *shape_gather_out); + fill_layer->setInput(0, *shape_gather_out); fill_layer->setInput(1, *valueTensor); at::Tensor delta_tensor = torch::zeros(inRank, util::TRTDataTypeToScalarType(in->getType())); auto deltaTensor = tensor_to_const(ctx, delta_tensor); From 6b77b721f140bb465477582c557284de53e62453 Mon Sep 17 00:00:00 2001 From: Michael Feliz Date: Thu, 30 Jun 2022 17:17:59 -0700 Subject: [PATCH 09/31] feat: Add support for aten::square --- core/conversion/converters/impl/element_wise.cpp | 12 ++++++++++++ .../core/conversion/converters/test_element_wise.cpp | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/core/conversion/converters/impl/element_wise.cpp b/core/conversion/converters/impl/element_wise.cpp index 32c7050289..8b08a5505a 100644 --- a/core/conversion/converters/impl/element_wise.cpp +++ b/core/conversion/converters/impl/element_wise.cpp @@ -390,6 +390,18 @@ auto element_wise_registrations TORCHTRT_UNUSED = LOG_DEBUG("Output tensor shape: " << out->getDimensions()); return true; }}) + .pattern( + {"aten::square(Tensor self) -> Tensor", + [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool { + auto self = args[0].ITensorOrFreeze(ctx); + auto mul = add_elementwise(ctx, nvinfer1::ElementWiseOperation::kPROD, self, self, util::node_info(n)); + TORCHTRT_CHECK(mul, "Unable to create mul layer from node: " << *n); + + mul->setName(util::node_info(n).c_str()); + auto out = ctx->AssociateValueAndTensor(n->outputs()[0], mul->getOutput(0)); + LOG_DEBUG("Output tensor shape: " << out->getDimensions()); + return true; + }}) .pattern( {"aten::mul.Tensor(Tensor self, Tensor other) -> Tensor", [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool { diff --git a/tests/core/conversion/converters/test_element_wise.cpp b/tests/core/conversion/converters/test_element_wise.cpp index 3ecfdb2019..6b1c26bbab 100644 --- a/tests/core/conversion/converters/test_element_wise.cpp +++ b/tests/core/conversion/converters/test_element_wise.cpp @@ -145,6 +145,14 @@ TEST(Converters, ATenMulConvertsCorrectly) { pointwise_test_helper(graph, false, true, {5}, {5}, false, at::kInt, at::kFloat); } +TEST(Converters, ATenSquareConvertsCorrectly) { + const auto graph = R"IR( + graph(%0 : Tensor): + %1 : Tensor = aten::square(%0) + return (%1))IR"; + pointwise_test_helper(graph, true); +} + TEST(Converters, ATenMulWithScalarConvertsCorrectly) { const auto graph = R"IR( graph(%0 : Tensor): From e74dbd7bbc7d6107976239572240d63be9d5907e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 19:56:33 +0000 Subject: [PATCH 10/31] chore(deps): bump @actions/core in /.github/actions/assigner Bumps [@actions/core](https://github.com/actions/toolkit/tree/HEAD/packages/core) from 1.8.2 to 1.9.1. - [Release notes](https://github.com/actions/toolkit/releases) - [Changelog](https://github.com/actions/toolkit/blob/main/packages/core/RELEASES.md) - [Commits](https://github.com/actions/toolkit/commits/HEAD/packages/core) --- updated-dependencies: - dependency-name: "@actions/core" dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/actions/assigner/package-lock.json | 33 ++++++++++++++++------ .github/actions/assigner/package.json | 2 +- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/actions/assigner/package-lock.json b/.github/actions/assigner/package-lock.json index 6807c4ed4e..9d077a8546 100644 --- a/.github/actions/assigner/package-lock.json +++ b/.github/actions/assigner/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "BSD-3-Clause", "dependencies": { - "@actions/core": "^1.8.2", + "@actions/core": "^1.9.1", "@actions/github": "^5.0.3", "fs": "^0.0.1-security", "js-yaml": "^4.1.0" @@ -22,11 +22,12 @@ } }, "node_modules/@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "dependencies": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "node_modules/@actions/github": { @@ -276,6 +277,14 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -298,11 +307,12 @@ }, "dependencies": { "@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "requires": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "@actions/github": { @@ -507,6 +517,11 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/.github/actions/assigner/package.json b/.github/actions/assigner/package.json index f6fa97add8..306ac4b4f6 100644 --- a/.github/actions/assigner/package.json +++ b/.github/actions/assigner/package.json @@ -19,7 +19,7 @@ "license": "BSD-3-Clause", "repository": "https://www.github.com/pytorch/TensorRT", "dependencies": { - "@actions/core": "^1.8.2", + "@actions/core": "^1.9.1", "@actions/github": "^5.0.3", "fs": "^0.0.1-security", "js-yaml": "^4.1.0" From 0591a8d137473417f57e27402bbbfd142d5ef397 Mon Sep 17 00:00:00 2001 From: Bo Wang Date: Thu, 18 Aug 2022 14:49:14 -0700 Subject: [PATCH 11/31] fix: fix misleading skipping partitioning msg Signed-off-by: Bo Wang --- core/compiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/compiler.cpp b/core/compiler.cpp index 7b58dbb2c1..898f2f1295 100644 --- a/core/compiler.cpp +++ b/core/compiler.cpp @@ -426,7 +426,7 @@ torch::jit::Module CompileGraph(const torch::jit::Module& mod, CompileSpec cfg) auto outputIsCollection = conversion::OutputIsCollection(g->block()); if (cfg.partition_info.enabled && (cfg.lower_info.forced_fallback_modules.size() == 0 && - cfg.partition_info.forced_fallback_operators.size() == 0 && isBlockConvertible)) { + cfg.partition_info.forced_fallback_operators.size() == 0 && isBlockConvertible) && !outputIsCollection) { LOG_INFO("Skipping partitioning since model is fully supported"); } From b6b690ca0d8d3a4c8c34a27eebf96ef0677e16ec Mon Sep 17 00:00:00 2001 From: Wei Date: Thu, 18 Aug 2022 18:10:31 -0700 Subject: [PATCH 12/31] Changes done internally at Facebook (#1288) bd46e8f292bf68fe6b87d2d5d206c89fda79a746 Shirong Wu Disable group ln fuse pass 6ce1d3bc19d75b266e99355c96daeff7054dcbf8 Wei Wei [fx2trt] set logging level to INFO at fx root 9d552dc3f69db9e4a249f80ef00803a9413e5d38 Wei Wei [fx2trt] change OSS method lower_to_trt() to compile() Co-authored-by: wwei6 --- .../getting_started_with_fx_path.rst | 6 +-- examples/fx/lower_example.py | 4 +- examples/fx/torchdynamo_example.py | 4 +- py/torch_tensorrt/_compile.py | 4 +- py/torch_tensorrt/fx/__init__.py | 4 ++ py/torch_tensorrt/fx/lower.py | 40 ++++++++++--------- py/torch_tensorrt/fx/lower_setting.py | 4 ++ .../fx/test/converters/acc_op/test_type_as.py | 3 +- .../fx/test/passes/test_graph_opts.py | 2 - .../fx/test/trt_lower/test_diagnostics.py | 2 - .../fx/tools/trt_profiler_sorted.py | 5 +++ 11 files changed, 46 insertions(+), 32 deletions(-) diff --git a/docsrc/tutorials/getting_started_with_fx_path.rst b/docsrc/tutorials/getting_started_with_fx_path.rst index d54f3d91af..eb39fc0eef 100644 --- a/docsrc/tutorials/getting_started_with_fx_path.rst +++ b/docsrc/tutorials/getting_started_with_fx_path.rst @@ -34,7 +34,7 @@ Torch-TensorRT (FX Path) is in ``Beta`` phase and always recommended to work wit Converting a PyTorch Model to TensorRT Engine --------------------------------------------- -In general, users are welcome to use the ``lower_to_trt()`` to finish the conversion from a model to tensorRT engine. It is a wrapper API that consists of the major steps needed to finish this converison. Please refer to ``lower_example.py`` file in ``examples/fx``. +In general, users are welcome to use the ``compile()`` to finish the conversion from a model to tensorRT engine. It is a wrapper API that consists of the major steps needed to finish this converison. Please refer to ``lower_example.py`` file in ``examples/fx``. In this section, we will go through an example to illustrate the major steps that FX path uses. Users can refer to ``fx2trt_example.py`` file in ``examples/fx``. @@ -60,9 +60,9 @@ symbolically traced variables cannot be used as inputs to control flow This means the model contains dynamic control flow. Please refer to section “Dynamic Control Flow” in `FX guide `_. * **Step 2: Build TensorRT engine** -There are `two different modes `_ for how TensorRT handles batch dimension, explicit batch dimension and implicit batch dimension. This mode was used by early versions of TensorRT, and is now deprecated but continues to be supported for backwards compatibility. In explicit batch mode, all dimensions are explicit and can be dynamic, that is their length can change at execution time. Many new features, such as dynamic shapes and loops, are available only in this mode. User can still choose to use implicit batch mode when they set ``explicit_batch_dimension=False`` in ``lower_to_trt()``. We do not recommend to use it since it will lack of support in future TensorRT versions. +There are `two different modes `_ for how TensorRT handles batch dimension, explicit batch dimension and implicit batch dimension. This mode was used by early versions of TensorRT, and is now deprecated but continues to be supported for backwards compatibility. In explicit batch mode, all dimensions are explicit and can be dynamic, that is their length can change at execution time. Many new features, such as dynamic shapes and loops, are available only in this mode. User can still choose to use implicit batch mode when they set ``explicit_batch_dimension=False`` in ``compile()``. We do not recommend to use it since it will lack of support in future TensorRT versions. -Explicit batch is the default mode and it must be set for dynamic shape. For most of vision task, user can choose to enable ``dynamic_batch`` in ``lower_to_trt()`` if they want to get the similar effects as implicit mode where only batch dimension changes. It has some requirements: +Explicit batch is the default mode and it must be set for dynamic shape. For most of vision task, user can choose to enable ``dynamic_batch`` in ``compile()`` if they want to get the similar effects as implicit mode where only batch dimension changes. It has some requirements: 1. Shapes of inputs, outputs and activations are fixed except batch dimension. 2. Inputs, outputs and activations have batch dimension as the major dimension. 3. All the operators in the model do not modify batch dimension (permute, transpose, split, etc.) or compute over batch dimension (sum, softmax, etc.). diff --git a/examples/fx/lower_example.py b/examples/fx/lower_example.py index 71f15a2f88..7f3b374f44 100644 --- a/examples/fx/lower_example.py +++ b/examples/fx/lower_example.py @@ -4,7 +4,7 @@ import torch import torchvision -from torch_tensorrt.fx.lower import lower_to_trt +from torch_tensorrt.fx.lower import compile from torch_tensorrt.fx.utils import LowerPrecision @@ -183,7 +183,7 @@ def run_configuration_benchmark( time = benchmark_torch_function(conf.batch_iter, lambda: module(*input)) elif not conf.jit: # Run lowering eager mode benchmark - lowered_module = lower_to_trt( + lowered_module = compile( module, input, max_batch_size=conf.batch_size, diff --git a/examples/fx/torchdynamo_example.py b/examples/fx/torchdynamo_example.py index 6bb93f6d6e..a2e7627800 100644 --- a/examples/fx/torchdynamo_example.py +++ b/examples/fx/torchdynamo_example.py @@ -5,7 +5,7 @@ import torch import torchdynamo import torchvision -from torch_tensorrt.fx.lower import lower_to_trt +from torch_tensorrt.fx.lower import compile from torch_tensorrt.fx.utils import LowerPrecision from torchdynamo.optimizations import backends @@ -197,7 +197,7 @@ def run_configuration_benchmark( if conf.trt: # Run lowering eager mode benchmark - lowered_module = lower_to_trt( + lowered_module = compile( module, input, max_batch_size=conf.batch_size, diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index f6487a4402..8b5f235531 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -7,7 +7,7 @@ from enum import Enum import torch_tensorrt.fx -from torch_tensorrt.fx.lower import lower_to_trt +import torch_tensorrt.fx.lower from torch_tensorrt.fx.utils import LowerPrecision @@ -140,7 +140,7 @@ def compile( else: raise ValueError(f"Precision {enabled_precisions} not supported on FX") - return lower_to_trt( + return torch_tensorrt.fx.lower.compile( module, inputs, lower_precision=lower_precision, diff --git a/py/torch_tensorrt/fx/__init__.py b/py/torch_tensorrt/fx/__init__.py index aeae62d86d..c1c42c446f 100644 --- a/py/torch_tensorrt/fx/__init__.py +++ b/py/torch_tensorrt/fx/__init__.py @@ -1,4 +1,6 @@ from .converters import * # noqa: F403 F401 +import logging + from .converter_registry import ( # noqa CONVERTERS, NO_EXPLICIT_BATCH_DIM_SUPPORT, @@ -9,3 +11,5 @@ from .input_tensor_spec import generate_input_specs, InputTensorSpec # noqa from .lower_setting import LowerSetting # noqa from .trt_module import TRTModule # noqa + +logging.basicConfig(level=logging.INFO) diff --git a/py/torch_tensorrt/fx/lower.py b/py/torch_tensorrt/fx/lower.py index 387b4db841..deeee14178 100644 --- a/py/torch_tensorrt/fx/lower.py +++ b/py/torch_tensorrt/fx/lower.py @@ -25,7 +25,7 @@ Input = Sequence[Any] -def lower_to_trt( +def compile( module: nn.Module, input, max_batch_size: int = 2048, @@ -216,28 +216,32 @@ def create( ) ) - @decorate_method(validate_inference(atol=1e-1, rtol=1e-1)) def __call__( self, module: nn.Module, inputs: Input, additional_inputs: Optional[Input] = None, ) -> nn.Module: - module.eval() - - if ( - self.lower_pass_manager_builder.lower_setting.lower_precision - == LowerPrecision.FP16 - ): - module.half() - inputs = tuple( - x.half() if x is not None and x.dtype == torch.float32 else x - for x in inputs + lower_setting = self.lower_pass_manager_builder.lower_setting + atol = lower_setting.correctness_atol + rtol = lower_setting.correctness_rtol + + @validate_inference(atol=atol, rtol=rtol) + def do_lower(module: nn.Module, inputs: Input) -> nn.Module: + module.eval() + if ( + self.lower_pass_manager_builder.lower_setting.lower_precision + == LowerPrecision.FP16 + ): + module.half() + inputs = tuple( + x.half() if x is not None and x.dtype == torch.float32 else x + for x in inputs + ) + pm = self.lower_pass_manager_builder.build_trt_lower_pipeline( + inputs, additional_inputs ) - pm = self.lower_pass_manager_builder.build_trt_lower_pipeline( - inputs, additional_inputs - ) - - lower_result = pm(module) + lower_result = pm(module) + return lower_result - return lower_result + return do_lower(module, inputs) diff --git a/py/torch_tensorrt/fx/lower_setting.py b/py/torch_tensorrt/fx/lower_setting.py index c1d02229e3..b4ad86caee 100644 --- a/py/torch_tensorrt/fx/lower_setting.py +++ b/py/torch_tensorrt/fx/lower_setting.py @@ -70,6 +70,8 @@ class LowerSetting(LowerSettingBasic): dynamic_batch: enable the dynamic shape in TRT with dim=-1 for the 1st dimension. tactic_sources: tactic sources for TensorRT kernel selection. Default to None, meaning all possible tactic sources. + correctness_atol: absolute tolerance for correctness check + correctness_rtol: relative tolerance for correctness check """ input_specs: List[InputTensorSpec] = dc.field(default_factory=list) @@ -90,3 +92,5 @@ class LowerSetting(LowerSettingBasic): opt_profile_replica: int = 1 dynamic_batch: bool = True tactic_sources: Optional[int] = None + correctness_atol: float = 0.1 + correctness_rtol: float = 0.1 diff --git a/py/torch_tensorrt/fx/test/converters/acc_op/test_type_as.py b/py/torch_tensorrt/fx/test/converters/acc_op/test_type_as.py index 0bfffd210f..1f3a39d836 100644 --- a/py/torch_tensorrt/fx/test/converters/acc_op/test_type_as.py +++ b/py/torch_tensorrt/fx/test/converters/acc_op/test_type_as.py @@ -1,5 +1,6 @@ -import torch import unittest + +import torch import torch_tensorrt.fx.tracer.acc_tracer.acc_ops as acc_ops from torch.testing._internal.common_utils import run_tests from torch_tensorrt.fx.tools.common_fx2trt import AccTestCase, InputTensorSpec diff --git a/py/torch_tensorrt/fx/test/passes/test_graph_opts.py b/py/torch_tensorrt/fx/test/passes/test_graph_opts.py index 9db4183e64..c91c456eb3 100644 --- a/py/torch_tensorrt/fx/test/passes/test_graph_opts.py +++ b/py/torch_tensorrt/fx/test/passes/test_graph_opts.py @@ -8,8 +8,6 @@ import torch_tensorrt.fx.tracer.acc_tracer.acc_tracer as acc_tracer from torch_tensorrt.fx.passes.graph_opts import common_subexpression_elimination -_LOGGER: logging.Logger = logging.getLogger(__name__) - _LOGGER: logging.Logger = logging.getLogger(__name__) diff --git a/py/torch_tensorrt/fx/test/trt_lower/test_diagnostics.py b/py/torch_tensorrt/fx/test/trt_lower/test_diagnostics.py index e23ab5dd81..3ce3b7ade8 100644 --- a/py/torch_tensorrt/fx/test/trt_lower/test_diagnostics.py +++ b/py/torch_tensorrt/fx/test/trt_lower/test_diagnostics.py @@ -10,8 +10,6 @@ import torch_tensorrt.fx.diagnostics as diag -_LOGGER: logging.Logger = logging.getLogger(__name__) - _LOGGER: logging.Logger = logging.getLogger(__name__) diff --git a/py/torch_tensorrt/fx/tools/trt_profiler_sorted.py b/py/torch_tensorrt/fx/tools/trt_profiler_sorted.py index 59d2f49042..ac0a02ac1d 100644 --- a/py/torch_tensorrt/fx/tools/trt_profiler_sorted.py +++ b/py/torch_tensorrt/fx/tools/trt_profiler_sorted.py @@ -37,6 +37,11 @@ def profile_trt_module( layer_info = json.loads(trt_mod.get_layer_info()) # pyre-ignore[29] shape_map = {} for layer in layer_info["Layers"]: + # if type is str, it means verbose_profile is off in interpreter.run() + # Theorectically, we can print profiling information without shape information + # but we choose to not print profiling information so we can use verbose_profile to control it + if type(layer) is str: + return name = layer["Name"] input_str = ", ".join( [str(x.get("Dimensions", "[]")) for x in layer.get("Inputs", [])] From 3b910caa73aab4e92941e34a26711211f686f643 Mon Sep 17 00:00:00 2001 From: Anurag Dixit Date: Thu, 18 Aug 2022 19:02:21 -0700 Subject: [PATCH 13/31] chore: Truncate long and double for ptq CPP path Signed-off-by: Anurag Dixit --- examples/int8/ptq/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/int8/ptq/main.cpp b/examples/int8/ptq/main.cpp index 4cc38148b6..e69227fb5a 100644 --- a/examples/int8/ptq/main.cpp +++ b/examples/int8/ptq/main.cpp @@ -50,6 +50,7 @@ torch::jit::Module compile_int8_model(const std::string& data_dir, torch::jit::M compile_spec.enabled_precisions.insert(torch::kI8); /// Use the TensorRT Entropy Calibrator compile_spec.ptq_calibrator = calibrator; + compile_spec.truncate_long_and_double = true; #ifdef SAVE_ENGINE std::cout << "Compiling graph to save as TRT engine (/tmp/engine_converted_from_jit.trt)" << std::endl; From 149033c529749f7be5bb3368fa694f2267c54a91 Mon Sep 17 00:00:00 2001 From: Anurag Dixit Date: Thu, 18 Aug 2022 19:03:37 -0700 Subject: [PATCH 14/31] chore: Fix dataloader in finetune_qat script Signed-off-by: Anurag Dixit --- examples/int8/training/vgg16/finetune_qat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/int8/training/vgg16/finetune_qat.py b/examples/int8/training/vgg16/finetune_qat.py index 48709d9f8a..6ec20a9a46 100644 --- a/examples/int8/training/vgg16/finetune_qat.py +++ b/examples/int8/training/vgg16/finetune_qat.py @@ -261,7 +261,7 @@ def main(): state = ckpt["state"] data = iter(training_dataloader) - images, _ = data.next() + images, _ = next(data) writer.add_graph(model, images.cuda()) writer.close() From ec2e2bfa9273b998b3df2156301c2e65521ae8d4 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 19 Aug 2022 13:52:36 -0700 Subject: [PATCH 15/31] docs: Update docgen task Signed-off-by: Naren Dasan Signed-off-by: Naren Dasan --- .github/workflows/docgen.yml | 51 +++++- docsrc/Makefile | 4 +- docsrc/WORKSPACE.docs | 0 docsrc/{tutorials => cli}/torchtrtc.rst | 0 .../getting_started_with_cpp_api.rst | 6 +- .../getting_started_with_python_api.rst | 17 +- .../installation.rst | 0 docsrc/index.rst | 67 ++++---- docsrc/py_api/fx.rst | 31 ++++ docsrc/py_api/torch_tensorrt.rst | 1 + .../getting_started_with_fx_path.rst | 43 ++--- docsrc/tutorials/notebooks.rst | 154 ++++++++++++++++++ py/requirements.txt | 5 +- py/torch_tensorrt/fx/__init__.py | 1 + 14 files changed, 308 insertions(+), 72 deletions(-) delete mode 100644 docsrc/WORKSPACE.docs rename docsrc/{tutorials => cli}/torchtrtc.rst (100%) rename docsrc/{tutorials => getting_started}/getting_started_with_cpp_api.rst (99%) rename docsrc/{tutorials => getting_started}/getting_started_with_python_api.rst (59%) rename docsrc/{tutorials => getting_started}/installation.rst (100%) create mode 100644 docsrc/py_api/fx.rst create mode 100644 docsrc/tutorials/notebooks.rst diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index c1d6f0944c..147625e51d 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -10,9 +10,9 @@ on: jobs: build-docs: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 container: - image: docker.pkg.github.com/pytorch/tensorrt/docgen:latest + image: nvidia/cuda:11.3.1-devel-ubuntu20.04 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -22,23 +22,64 @@ jobs: rm -rf /usr/share/dotnet rm -rf /opt/ghc rm -rf "/usr/local/share/boost" + rm -rf /usr/local/cuda/cuda-* + - name: Install Python + run: + apt update + apt install -y gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc + mkdir -p /opt/circleci + git clone https://github.com/pyenv/pyenv.git /opt/circleci/.pyenv + export PYENV_ROOT="/opt/circleci/.pyenv" + export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" + pyenv install 3.9.4 + pyenv global 3.9.4 + python3 -m pip install --upgrade pip + python3 -m pip install wheel - uses: actions/checkout@v2 with: ref: ${{github.head_ref}} - name: Get HEAD SHA id: vars run: echo "::set-output name=sha::$(git rev-parse --short HEAD)" + - name: Get Bazel version + id: bazel_info + run: echo "::set-output name=version::$(cat .bazelversion)" + - name: Install Bazel + run: | + wget -q https://github.com/bazelbuild/bazel/releases/download/${{ steps.bazel_info.outputs.version }}/bazel-${{ steps.bazel_info.outputs.version }}-linux-x86_64 -O /usr/bin/bazel + chmod a+x /usr/bin/bazel + - name: Install cudnn + tensorrt + run: | + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin + mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 536F8F1DE80F6A35 + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + apt-get update + apt-get install -y libcudnn8 libcudnn8-dev + + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + apt-get update + + apt-get install -y libnvinfer8 libnvinfer-plugin8 libnvinfer-dev libnvinfer-plugin-dev + - name: Install Torch + run: | + python3 -m pip install -r py/requirements.txt - name: Build Python Package run: | - cp docker/WORKSPACE.docker WORKSPACE + cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py - python3 setup.py install + pip install -e . + cd .. - name: Generate New Docs run: | cd docsrc - pip3 install -r requirements.txt + python3 -m pip install -r requirements.txt python3 -c "import torch_tensorrt; print(torch_tensorrt.__version__)" make html + cd .. - uses: stefanzweifel/git-auto-commit-action@v4 with: # Required diff --git a/docsrc/Makefile b/docsrc/Makefile index 18c52c578b..0ea6796ed8 100644 --- a/docsrc/Makefile +++ b/docsrc/Makefile @@ -37,8 +37,8 @@ endif rm -rf $(SOURCEDIR)/_tmp html: - mkdir -p $(SOURCEDIR)/_notebooks - cp -r $(SOURCEDIR)/../notebooks/*.ipynb $(SOURCEDIR)/_notebooks +# mkdir -p $(SOURCEDIR)/_notebooks +# cp -r $(SOURCEDIR)/../notebooks/*.ipynb $(SOURCEDIR)/_notebooks @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) mkdir -p $(DESTDIR) cp -r $(BUILDDIR)/html/* $(DESTDIR) diff --git a/docsrc/WORKSPACE.docs b/docsrc/WORKSPACE.docs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docsrc/tutorials/torchtrtc.rst b/docsrc/cli/torchtrtc.rst similarity index 100% rename from docsrc/tutorials/torchtrtc.rst rename to docsrc/cli/torchtrtc.rst diff --git a/docsrc/tutorials/getting_started_with_cpp_api.rst b/docsrc/getting_started/getting_started_with_cpp_api.rst similarity index 99% rename from docsrc/tutorials/getting_started_with_cpp_api.rst rename to docsrc/getting_started/getting_started_with_cpp_api.rst index 41b8a448c0..7f7f60a669 100644 --- a/docsrc/tutorials/getting_started_with_cpp_api.rst +++ b/docsrc/getting_started/getting_started_with_cpp_api.rst @@ -1,7 +1,7 @@ -.. _getting_started: +.. _getting_started_cpp: -Getting Started with C++ -======================== +Using Torch-TensorRT in C++ +============================== If you haven't already, acquire a tarball of the library by following the instructions in :ref:`Installation` diff --git a/docsrc/tutorials/getting_started_with_python_api.rst b/docsrc/getting_started/getting_started_with_python_api.rst similarity index 59% rename from docsrc/tutorials/getting_started_with_python_api.rst rename to docsrc/getting_started/getting_started_with_python_api.rst index 62d5d07b80..fece176156 100644 --- a/docsrc/tutorials/getting_started_with_python_api.rst +++ b/docsrc/getting_started/getting_started_with_python_api.rst @@ -3,8 +3,16 @@ Using Torch-TensorRT in Python ******************************* -Torch-TensorRT Python API accepts a ```torch.nn.Module`` as an input. Under the hood, it uses ``torch.jit.script`` to convert the input module into a -TorchScript module. To compile your input ```torch.nn.Module`` with Torch-TensorRT, all you need to do is provide the module and inputs +The Torch-TensorRT Python API supports a number of unique usecases compared to the CLI and C++ APIs which solely support TorchScript compilation. + +Torch-TensorRT Python API can accept a ``torch.nn.Module``, ``torch.jit.ScriptModule``, or ``torch.fx.GraphModule`` as an input. +Depending on what is provided one of the two frontends (TorchScript or FX) will be selected to compile the module. Provided the +module type is supported, users may explicitly set which frontend they would like to use using the ``ir`` flag for ``compile``. +If given a ``torch.nn.Module`` and the ``ir`` flag is set to either ``default`` or ``torchscript`` the module will be run through +``torch.jit.script`` to convert the input module into a TorchScript module. + + +To compile your input ``torch.nn.Module`` with Torch-TensorRT, all you need to do is provide the module and inputs to Torch-TensorRT and you will be returned an optimized TorchScript module to run or add into another PyTorch module. Inputs is a list of ``torch_tensorrt.Input`` classes which define input's shape, datatype and memory format. You can also specify settings such as operating precision for the engine or target device. After compilation you can save the module just like any other module @@ -46,6 +54,5 @@ to load in a deployment application. In order to load a TensorRT/TorchScript mod input_data = input_data.to("cuda").half() result = trt_ts_module(input_data) -Torch-TensorRT python API also provides ``torch_tensorrt.ts.compile`` which accepts a TorchScript module as input. -The torchscript module can be obtained via scripting or tracing (refer to :ref:`creating_torchscript_module_in_python`). ``torch_tensorrt.ts.compile`` accepts a Torchscript module -and a list of ``torch_tensorrt.Input`` classes. +Torch-TensorRT Python API also provides ``torch_tensorrt.ts.compile`` which accepts a TorchScript module as input and ``torch_tensorrt.fx.compile`` which accepts a FX GraphModule as input. + diff --git a/docsrc/tutorials/installation.rst b/docsrc/getting_started/installation.rst similarity index 100% rename from docsrc/tutorials/installation.rst rename to docsrc/getting_started/installation.rst diff --git a/docsrc/index.rst b/docsrc/index.rst index b12d6cc1f4..c0de8dd400 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -22,46 +22,43 @@ More Information / System Architecture: Getting Started ---------------- * :ref:`installation` -* :ref:`getting_started` +* :ref:`getting_started_with_python_api` +* :ref:`getting_started_cpp` + +.. toctree:: + :caption: Getting Started + :maxdepth: 1 + :hidden: + + getting_started/installation + getting_started/getting_started_with_python_api + getting_started/getting_started_with_cpp_api + + +Tutorials +------------ +* :ref:`creating_a_ts_mod` +* :ref:`getting_started_with_fx` * :ref:`ptq` -* :ref:`torchtrtc` -* :ref:`use_from_pytorch` * :ref:`runtime` -* :ref:`using_dla` * :ref:`serving_torch_tensorrt_with_triton` -* :ref:`user_guide` +* :ref:`use_from_pytorch` +* :ref:`using_dla` +* :ref:`notebooks` .. toctree:: - :caption: Getting Started + :caption: Tutorials :maxdepth: 1 :hidden: - tutorials/installation - tutorials/getting_started_with_cpp_api - tutorials/getting_started_with_python_api tutorials/creating_torchscript_module_in_python + tutorials/getting_started_with_fx_path tutorials/ptq - tutorials/torchtrtc - tutorials/use_from_pytorch tutorials/runtime - tutorials/using_dla tutorials/serving_torch_tensorrt_with_triton - tutorials/getting_started_with_fx_path - -.. toctree:: - :caption: Notebooks - :maxdepth: 1 - :hidden: - - _notebooks/CitriNet-example - _notebooks/dynamic-shapes - _notebooks/EfficientNet-example - _notebooks/Hugging-Face-BERT - _notebooks/lenet-getting-started - _notebooks/Resnet50-example - _notebooks/ssd-object-detection-demo - _notebooks/vgg-qat - + tutorials/use_from_pytorch + tutorials/using_dla + tutorials/notebooks Python API Documenation ------------------------ @@ -69,6 +66,7 @@ Python API Documenation * :ref:`torch_tensorrt_logging_py` * :ref:`torch_tensorrt_ptq_py` * :ref:`torch_tensorrt_ts_py` +* :ref:`torch_tensorrt_fx_py` .. toctree:: :caption: Python API Documenation @@ -79,6 +77,7 @@ Python API Documenation py_api/logging py_api/ptq py_api/ts + py_api/fx C++ API Documenation ---------------------- @@ -99,6 +98,18 @@ C++ API Documenation _cpp_api/namespace_torch_tensorrt__torchscript _cpp_api/namespace_torch_tensorrt__ptq +CLI Documentation +--------------------- +* :ref:`torchtrtc` + +.. toctree:: + :caption: CLI Documenation + :maxdepth: 0 + :hidden: + + cli/torchtrtc + + Contributor Documentation -------------------------------- * :ref:`system_overview` diff --git a/docsrc/py_api/fx.rst b/docsrc/py_api/fx.rst new file mode 100644 index 0000000000..8ce591f5ee --- /dev/null +++ b/docsrc/py_api/fx.rst @@ -0,0 +1,31 @@ +.. _torch_tensorrt_fx_py: + +torch_tensorrt.fx +=================== + +.. currentmodule:: torch_tensorrt.fx + +.. automodule torch_tensorrt.ts + :undoc-members: + +.. automodule:: torch_tensorrt.fx + :members: + :undoc-members: + :show-inheritance: + +Functions +------------ + +.. autofunction:: compile + + +Classes +-------- + +.. autoclass:: TRTModule + +.. autoclass:: InputTensorSpec + +.. autoclass:: TRTInterpreter + +.. autoclass:: TRTInterpreterResult \ No newline at end of file diff --git a/docsrc/py_api/torch_tensorrt.rst b/docsrc/py_api/torch_tensorrt.rst index 03f6d8101b..2ccf7c60e2 100644 --- a/docsrc/py_api/torch_tensorrt.rst +++ b/docsrc/py_api/torch_tensorrt.rst @@ -57,3 +57,4 @@ Submodules logging ptq ts + fx diff --git a/docsrc/tutorials/getting_started_with_fx_path.rst b/docsrc/tutorials/getting_started_with_fx_path.rst index eb39fc0eef..d7dc9b0166 100644 --- a/docsrc/tutorials/getting_started_with_fx_path.rst +++ b/docsrc/tutorials/getting_started_with_fx_path.rst @@ -1,42 +1,29 @@ -.. user_guide: -Torch-TensorRT (FX Path) User Guide -======================== -Torch-TensorRT (FX Path) is a tool that can convert a PyTorch model through torch.FX to an TensorRT engine optimized targeting running on Nvidia GPUs. TensorRT is the inference engine developed by Nvidia which composed of various kinds of optimization including kernel fusion, graph optimization, low precision, etc.. -This tool is developed in Python environment providing most usability to researchers and engineers. There are a few stages that a user want to use this tool and we will introduce them here. - - -Installation ------------- -Torch-TensorRT (FX Path) is in ``Beta`` phase and always recommended to work with PyTorch nightly. +.. _getting_started_with_fx: +Torch-TensorRT (FX Frontend) User Guide +======================== +Torch-TensorRT (FX Frontend) is a tool that can convert a PyTorch model through ``torch.fx`` to an +TensorRT engine optimized targeting running on Nvidia GPUs. TensorRT is the inference engine +developed by NVIDIA which composed of various kinds of optimization including kernel fusion, +graph optimization, low precision, etc.. This tool is developed in Python environment which allows this +workflow to be very accessible to researchers and engineers. There are a few stages that a +user want to use this tool and we will introduce them here. -* Method 1. Follow the instrucions for Torch-TensorRT -* Method 2. To install FX path only (Python path) and avoid the C++ build for torchscript path +> Torch-TensorRT (FX Frontend) is in ``Beta`` and currently it is recommended to work with PyTorch nightly. .. code-block:: shell - $ conda create --name python_env python=3.8 - $ conda activate python_env - - # Recommend to install PyTorch 1.12 and later - $ conda install pytorch torchvision torchtext cudatoolkit=11.3 -c pytorch-nightly - - # Install TensorRT python package - $ pip3 install nvidia-pyindex - $ pip3 install nvidia-tensorrt==8.2.4.2 - $ git clone https://github.com/pytorch/TensorRT.git - $ cd TensorRT/py && python setup.py install --fx-only && cd .. - - $ pyton -c "import torch_tensorrt.fx" # Test an example by $ python py/torch_tensorrt/fx/example/lower_example.py Converting a PyTorch Model to TensorRT Engine --------------------------------------------- -In general, users are welcome to use the ``compile()`` to finish the conversion from a model to tensorRT engine. It is a wrapper API that consists of the major steps needed to finish this converison. Please refer to ``lower_example.py`` file in ``examples/fx``. +In general, users are welcome to use the ``compile()`` to finish the conversion from a model to tensorRT engine. It is a +wrapper API that consists of the major steps needed to finish this converison. Please refer to ``lower_example.py`` file in ``examples/fx``. -In this section, we will go through an example to illustrate the major steps that FX path uses. Users can refer to ``fx2trt_example.py`` file in ``examples/fx``. +In this section, we will go through an example to illustrate the major steps that fx path uses. +Users can refer to ``fx2trt_example.py`` file in ``examples/fx``. * **Step 1: Trace the model with acc_tracer** Acc_tracer is a tracer inheritated from FX tracer. It comes with args normalizer to convert all args to kwargs and pass to TRT converters. @@ -276,7 +263,7 @@ In the custom mapper function, we construct an acc op node and return it. The no The last step would be *adding unit test* for the new acc op or mapper function we added. The place to add the unit test is here `test_acc_tracer.py `_. -* **Step 2. Add a new fx2trt converter** +* **Step 2. Add a new converter** All the developed converters for acc ops are all in `acc_op_converter.py `_. It could give you a good example of how the converter is added. diff --git a/docsrc/tutorials/notebooks.rst b/docsrc/tutorials/notebooks.rst new file mode 100644 index 0000000000..df903fc353 --- /dev/null +++ b/docsrc/tutorials/notebooks.rst @@ -0,0 +1,154 @@ +.. _notebooks: + +Example notebooks +=================== + +There exists a number of notebooks which cover specific using specific features and models +with Torch-TensorRT + +Notebooks +------------ + +Compiling CitriNet with Torch-TensorRT +******************************************** + +Citrinet is an acoustic model used for the speech to text recognition task. It is a version +of QuartzNet that extends ContextNet, utilizing subword encoding (via Word Piece tokenization) +and Squeeze-and-Excitation(SE) mechanism and are therefore smaller than QuartzNet models. CitriNet +models take in audio segments and transcribe them to letter, byte pair, or word piece sequences. + +This notebook demonstrates the steps for optimizing a pretrained CitriNet model with Torch-TensorRT, +and running it to test the speedup obtained. + +* `Torch-TensorRT Getting Started - CitriNet `_ + + +Compiling EfficentNet with Torch-TensorRT +******************************************** + +EfficentNet is a feedforward CNN designed to achieve better performance and accuracy than alternative architectures +by using a "scaling method that uniformly scales all dimensions of depth/width/resolution using a simple yet highly effective compound coefficient". + +This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT, +and running it to test the speedup obtained. + +* `Torch-TensorRT Getting Started - EfficientNet-B0 `_ + + +Masked Language Modeling (MLM) with Hugging Face BERT Transformer accelerated by Torch-TensorRT +************************************************************************************************* + +"BERT is a transformer model pretrained on a large corpus of English data in a self-supervised fashion. +This way, the model learns an inner representation of the English language that can then be used to extract +features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train +a standard classifier using the features produced by the BERT model as inputs." (https://huggingface.co/bert-base-uncased) + +This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT, +and running it to test the speedup obtained. + +* `Masked Language Modeling (MLM) with Hugging Face BERT Transformer `_ + + +Serving a model in C++ using Torch-TensorRT +********************************************** + +This example shows how you can load a pretrained ResNet-50 model, convert it to a Torch-TensorRT +optimized model (via the Torch-TensorRT Python API), save the model as a torchscript module, and +then finally load and serve the model with the PyTorch C++ API. + +* `ResNet C++ Serving Example `_ + + +Compiling ResNet50 with Torch-TensorRT +******************************************** + +This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a +pretrained ResNet-50 network, and running it to test the speedup obtained. + +* `Torch-TensorRT Getting Started - ResNet 50 `_ + + +Using Dynamic Shapes with Torch-TensorRT +******************************************** + +Making use of Dynamic Shaped Tensors in Torch TensorRT is quite simple. Let's say you are +using the ``torch_tensorrt.compile(...)`` function to compile a torchscript module. One +of the args in this function in this function is ``input``: which defines an input to a +module in terms of expected shape, data type and tensor format: ``torch_tensorrt.Input.`` + +For the purposes of this walkthrough we just need three kwargs: `min_shape`, `opt_shape`` and `max_shape`. + +.. code-block:: py + + torch_tensorrt.Input( + min_shape=(1, 224, 224, 3), + opt_shape=(1, 512, 512, 3), + max_shape=(1, 1024, 1024, 3), + dtype=torch.int32 + format=torch.channel_last + ) + ... + +In this example, we are going to use a simple ResNet model to demonstrate the use of the API. + +* `Torch-TensorRT - Using Dynamic Shapes `_ + +Using the FX Frontend with Torch-TensorRT +******************************************** + +The purpose of this example is to demostrate the overall flow of lowering a PyTorch model to TensorRT +conveniently with using FX. + +* `Using the FX Frontend with Torch-TensorRT `_ + + +Compiling a PyTorch model using FX Frontend with Torch-TensorRT +******************************************************************* + +The purpose of this example is to demonstrate the overall flow of lowering a PyTorch +model to TensorRT via FX with existing FX based tooling + +* `Compiling a PyTorch model using FX Frontend with Torch-TensorRT `_ + + +Compiling LeNet with Torch-TensorRT +******************************************************************* + +This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a simple LeNet network. + +* `Torch-TensorRT Getting Started - LeNet `_ + + +Accelerate Deep Learning Models using Quantization in Torch-TensorRT +******************************************************************* + +Model Quantization is a popular way of optimization which reduces the size of models thereby +accelerating inference, also opening up the possibilities of deployments on devices with lower +computation power such as Jetson. Simply put, quantization is a process of mapping input values + from a larger set to output values in a smaller set. In this notebook, we illustrate the workflow + that you can adopt while quantizing a deep learning model in Torch-TensorRT. The notebook takes + you through an example of Mobilenetv2 for a classification task on a subset of Imagenet Dataset + called Imagenette which has 10 classes. + +* `Accelerate Deep Learning Models using Quantization in Torch-TensorRT `_ + + +Object Detection with Torch-TensorRT (SSD) +******************************************************************* + +This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a pretrained SSD network, and running it to test the speedup obtained. + +* `Object Detection with Torch-TensorRT (SSD) `_ + + +Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT +***************************************************************************** + +Quantization Aware training (QAT) simulates quantization during training by +quantizing weights and activation layers. This will help to reduce the loss in +accuracy when we convert the network trained in FP32 to INT8 for faster inference. +QAT introduces additional nodes in the graph which will be used to learn the dynamic +ranges of weights and activation layers. In this notebook, we illustrate the following +steps from training to inference of a QAT model in Torch-TensorRT. + +* `Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT `_ diff --git a/py/requirements.txt b/py/requirements.txt index 29bd0f92eb..5254a658a4 100644 --- a/py/requirements.txt +++ b/py/requirements.txt @@ -1,3 +1,6 @@ +numpy +pybind11==2.6.2 --extra-index-url https://download.pytorch.org/whl/cu113 torch==1.12.0 -pybind11==2.6.2 +--extra-index-url https://pypi.ngc.nvidia.com +nvidia-tensorrt==8.4.1.5 diff --git a/py/torch_tensorrt/fx/__init__.py b/py/torch_tensorrt/fx/__init__.py index c1c42c446f..493d749d09 100644 --- a/py/torch_tensorrt/fx/__init__.py +++ b/py/torch_tensorrt/fx/__init__.py @@ -11,5 +11,6 @@ from .input_tensor_spec import generate_input_specs, InputTensorSpec # noqa from .lower_setting import LowerSetting # noqa from .trt_module import TRTModule # noqa +from .lower import compile logging.basicConfig(level=logging.INFO) From 12f39acce5f4a15dd3048f54d5943e4eccbfac1c Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 19 Aug 2022 14:07:02 -0700 Subject: [PATCH 16/31] chore: apply linting Signed-off-by: Naren Dasan Signed-off-by: Naren Dasan --- core/compiler.cpp | 3 ++- tests/core/conversion/converters/test_select.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/compiler.cpp b/core/compiler.cpp index 898f2f1295..bf128b714a 100644 --- a/core/compiler.cpp +++ b/core/compiler.cpp @@ -426,7 +426,8 @@ torch::jit::Module CompileGraph(const torch::jit::Module& mod, CompileSpec cfg) auto outputIsCollection = conversion::OutputIsCollection(g->block()); if (cfg.partition_info.enabled && (cfg.lower_info.forced_fallback_modules.size() == 0 && - cfg.partition_info.forced_fallback_operators.size() == 0 && isBlockConvertible) && !outputIsCollection) { + cfg.partition_info.forced_fallback_operators.size() == 0 && isBlockConvertible) && + !outputIsCollection) { LOG_INFO("Skipping partitioning since model is fully supported"); } diff --git a/tests/core/conversion/converters/test_select.cpp b/tests/core/conversion/converters/test_select.cpp index d77fa37d40..e9405c0155 100644 --- a/tests/core/conversion/converters/test_select.cpp +++ b/tests/core/conversion/converters/test_select.cpp @@ -865,7 +865,7 @@ TEST(Converters, ScatterValueConvertsCorrectly) { %5 : NoneType = prim::Constant() %6 : bool = prim::Constant[value=0]() %7 : int = prim::Constant[value=4]() - %index : Tensor = aten::to(%index.1, %7, %6, %6, %5) + %index : Tensor = aten::to(%index.1, %7, %6, %6, %5) %10 : Tensor = aten::scatter(%data, %dim, %index, %value) return (%10))IR"; @@ -900,7 +900,7 @@ TEST(Converters, ScatterSrcConvertsCorrectly) { %5 : NoneType = prim::Constant() %6 : bool = prim::Constant[value=0]() %7 : int = prim::Constant[value=4]() - %index : Tensor = aten::to(%index.1, %7, %6, %6, %5) + %index : Tensor = aten::to(%index.1, %7, %6, %6, %5) %10 : Tensor = aten::scatter(%data, %dim, %index, %src) return (%10))IR"; From cfd816f04eecadd81ec62ba6051672cef5b111d2 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:18:38 -0700 Subject: [PATCH 17/31] Update docgen.yml --- .github/workflows/docgen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 147625e51d..d967248e11 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -12,7 +12,7 @@ jobs: build-docs: runs-on: ubuntu-20.04 container: - image: nvidia/cuda:11.3.1-devel-ubuntu20.04 + image: ghcr.io/pytorch/tensorrt/docgen:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -88,4 +88,4 @@ jobs: file_pattern: docs/ commit_user_name: Torch-TensorRT Github Bot commit_user_email: torch-tensorrt.github.bot@nvidia.com - commit_author: Torch-TensorRT Github Bot \ No newline at end of file + commit_author: Torch-TensorRT Github Bot From 5d8b2a73e3f53a2b8ec8c6739812c622c6a59a28 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:23:52 -0700 Subject: [PATCH 18/31] Update docgen.yml --- .github/workflows/docgen.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index d967248e11..9d353eb5aa 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -23,10 +23,12 @@ jobs: rm -rf /opt/ghc rm -rf "/usr/local/share/boost" rm -rf /usr/local/cuda/cuda-* - - name: Install Python - run: + - name: Install base deps + run: | apt update apt install -y gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc + - name: Install Python + run: | mkdir -p /opt/circleci git clone https://github.com/pyenv/pyenv.git /opt/circleci/.pyenv export PYENV_ROOT="/opt/circleci/.pyenv" From 7215a67d0c5e4f117e36a749da9f13bafe4152b7 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:29:47 -0700 Subject: [PATCH 19/31] Update docgen.yml --- .github/workflows/docgen.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 9d353eb5aa..0b376f7461 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -26,7 +26,7 @@ jobs: - name: Install base deps run: | apt update - apt install -y gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc + apt install -y software-properties-common gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc - name: Install Python run: | mkdir -p /opt/circleci @@ -52,6 +52,7 @@ jobs: chmod a+x /usr/bin/bazel - name: Install cudnn + tensorrt run: | + apt-get update wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub From 69ba6f8ff2cb68b5af962d9944bef3ebb902cfda Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:34:31 -0700 Subject: [PATCH 20/31] Update docgen.yml --- .github/workflows/docgen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 0b376f7461..cc9793f0c1 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -26,7 +26,7 @@ jobs: - name: Install base deps run: | apt update - apt install -y software-properties-common gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc + DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc - name: Install Python run: | mkdir -p /opt/circleci From 110d425089c88cf10f299bef52b9bd962253175d Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:52:12 -0700 Subject: [PATCH 21/31] Update docgen.yml --- .github/workflows/docgen.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index cc9793f0c1..21872ade60 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -69,16 +69,25 @@ jobs: apt-get install -y libnvinfer8 libnvinfer-plugin8 libnvinfer-dev libnvinfer-plugin-dev - name: Install Torch run: | + export PYENV_ROOT="/opt/circleci/.pyenv" + export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" + pyenv global 3.9.4 python3 -m pip install -r py/requirements.txt - name: Build Python Package run: | cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py - pip install -e . + export PYENV_ROOT="/opt/circleci/.pyenv" + export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" + pyenv global 3.9.4 + python3 -m pip install -e . cd .. - name: Generate New Docs run: | cd docsrc + export PYENV_ROOT="/opt/circleci/.pyenv" + export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" + pyenv global 3.9.4 python3 -m pip install -r requirements.txt python3 -c "import torch_tensorrt; print(torch_tensorrt.__version__)" make html From 7836ce233816a9fd3b6dfae920f0f2c27389bb4f Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 15:18:55 -0700 Subject: [PATCH 22/31] Update docgen.yml --- .github/workflows/docgen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 21872ade60..b2a9d77290 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -80,7 +80,7 @@ jobs: export PYENV_ROOT="/opt/circleci/.pyenv" export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" pyenv global 3.9.4 - python3 -m pip install -e . + python3 setup.py install cd .. - name: Generate New Docs run: | From 10325f14a58fa495a937fbb7f4ac491e4c4354c6 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Fri, 19 Aug 2022 15:31:18 -0700 Subject: [PATCH 23/31] Update docgen.yml --- .github/workflows/docgen.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index b2a9d77290..8a3fd24717 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -27,6 +27,7 @@ jobs: run: | apt update DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc + git config --global --add safe.directory '*' - name: Install Python run: | mkdir -p /opt/circleci From d7fd6916444fd77222819f9154dea6386db5b4ed Mon Sep 17 00:00:00 2001 From: Wei Date: Mon, 22 Aug 2022 10:52:43 -0700 Subject: [PATCH 24/31] update fx notebook (#1297) --- ...ng_started_with_fx_path_lower_to_trt.ipynb | 183 +++++++++++++----- 1 file changed, 137 insertions(+), 46 deletions(-) diff --git a/notebooks/getting_started_with_fx_path_lower_to_trt.ipynb b/notebooks/getting_started_with_fx_path_lower_to_trt.ipynb index 5ef957fa36..8e480903ab 100644 --- a/notebooks/getting_started_with_fx_path_lower_to_trt.ipynb +++ b/notebooks/getting_started_with_fx_path_lower_to_trt.ipynb @@ -10,14 +10,14 @@ "bento/extensions/theme/main.css": true }, "kernelspec": { - "display_name": "accelerators", + "display_name": "dper3_pytorch (cinder)", "language": "python", - "name": "bento_kernel_accelerators", + "name": "bento_kernel_dper3_pytorch_cinder", "metadata": { - "kernel_name": "bento_kernel_accelerators", - "nightly_builds": true, + "kernel_name": "bento_kernel_dper3_pytorch_cinder", + "nightly_builds": false, "fbpkg_supported": true, - "cinder_runtime": false, + "cinder_runtime": true, "is_prebuilt": true } }, @@ -32,10 +32,10 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3" }, - "last_server_session_id": "c6f6ab3c-9274-41e7-8592-b1b583442e00", - "last_kernel_id": "fcbf3a69-76a4-4730-9b41-bcd0b24729ca", - "last_base_url": "https://devgpu005.ftw6.facebook.com:8093/", - "last_msg_id": "e28f842c-f32dde25c1b80ef7d423dfee_407", + "last_server_session_id": "24a1a10c-29aa-4e2b-a11f-2b5108fc1e58", + "last_kernel_id": "5f014373-151c-4ee8-8939-4daab994d202", + "last_base_url": "https://devgpu005.ftw6.facebook.com:8091/", + "last_msg_id": "687e81e8-4414f32c89cd026dd1ea3fd9_139", "outputWidgetContext": {} }, "nbformat": 4, @@ -58,14 +58,14 @@ { "cell_type": "code", "metadata": { - "originalKey": "7909785f-b9b4-41dd-82af-c144b879df39", + "originalKey": "7db2accc-9fa4-4a1e-8142-d887f2947bcd", "showInput": true, "customInput": null, "collapsed": false, - "requestMsgId": "7db2accc-9fa4-4a1e-8142-d887f2947bcd", + "requestMsgId": "b5d8efce-0963-4074-bc9d-e8e1a78fd424", "customOutput": null, - "executionStartTime": 1656395936225, - "executionStopTime": 1656395937851 + "executionStartTime": 1661189891682, + "executionStopTime": 1661189891856 }, "source": [ "import typing as t\n", @@ -74,10 +74,10 @@ "\n", "import torch\n", "import torchvision\n", - "from torch_tensorrt.fx.lower import lower_to_trt\n", + "from torch_tensorrt.fx.lower import compile\n", "from torch_tensorrt.fx.utils import LowerPrecision" ], - "execution_count": 4, + "execution_count": 9, "outputs": [] }, { @@ -98,16 +98,16 @@ { "cell_type": "code", "metadata": { - "originalKey": "a4455135-8633-4d2d-bdd3-6435a4a9f4dd", + "originalKey": "2835fffa-cc50-479a-9080-c4f7002c0726", "showInput": true, "customInput": null, "code_folding": [], "hidden_ranges": [], "collapsed": false, - "requestMsgId": "2835fffa-cc50-479a-9080-c4f7002c0726", + "requestMsgId": "6ea72dbf-dbfe-451e-8613-15f87e34a1a5", "customOutput": null, - "executionStartTime": 1656398717455, - "executionStopTime": 1656398717662 + "executionStartTime": 1661189260550, + "executionStopTime": 1661189262039 }, "source": [ "@dataclass\n", @@ -159,24 +159,39 @@ " f\"Accuracy: {self.accuracy_res} (rtol={self.conf.accuracy_rtol})\"\n", " )" ], - "execution_count": 22, - "outputs": [] + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 102740.872 _utils_internal.py:179] NCCL_DEBUG env var is set to None\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 102740.873 _utils_internal.py:188] NCCL_DEBUG is INFO from /etc/nccl.conf\n" + ] + } + ] }, { "cell_type": "markdown", "metadata": { "originalKey": "3e462cf6-d282-402d-955b-a3ecb400bf0b", - "showInput": true, + "showInput": false, "customInput": null, "code_folding": [], "hidden_ranges": [] }, "source": [ "Run FX path lowering and benchmark the given model according to the specified benchmark configuration. Prints the benchmark result for each configuration at the end of the run. `benchmark_torch_function` is the actual function that computes the fixed number of iterations of functions runs.\n", - "The FX path lowering and TensorRT engine creation is integrated into `low_to_trt()` API which is defined in `fx/lower.py` file.\n", + "The FX path lowering and TensorRT engine creation is integrated into `compile()` API which is defined in `fx/lower.py` file.\n", "It is good to list it out and show the usage of it. It takes in original module, input and lowering setting, run lowering workflow to turn module into a executable TRT engine \n", "```\n", - "def lower_to_trt(\n", + "def compile(\n", " module: nn.Module,\n", " input: ,\n", " max_batch_size: int = 2048,\n", @@ -212,22 +227,18 @@ { "cell_type": "code", "metadata": { - "originalKey": "91333212-7f6d-4bde-a248-44d485e83e5e", + "originalKey": "3002935b-b95a-4a08-a57f-f7a35485af5b", "showInput": true, "customInput": null, "code_folding": [], "hidden_ranges": [], "collapsed": false, - "requestMsgId": "3002935b-b95a-4a08-a57f-f7a35485af5b", + "requestMsgId": "dc73f2d0-427b-4f71-bec1-b118cc5642d0", "customOutput": null, - "executionStartTime": 1656397903207, - "executionStopTime": 1656397964752 + "executionStartTime": 1661189697773, + "executionStopTime": 1661189753875 }, "source": [ - "test_model = torchvision.models.resnet18(pretrained=True)\n", - "input = [torch.rand(128, 3, 224, 224)] \n", - "benchmark(test_model, input, 50, 128)\n", - "\n", "def benchmark_torch_function(iters: int, f, *args) -> float:\n", " \"\"\"Estimates the average time duration for a single inference call in second\n", "\n", @@ -266,7 +277,7 @@ " time = benchmark_torch_function(conf.batch_iter, lambda: module(*input))\n", " elif not conf.jit:\n", " # Run lowering eager mode benchmark\n", - " lowered_module = lower_to_trt(\n", + " lowered_module = compile(\n", " module,\n", " input,\n", " max_batch_size=conf.batch_size,\n", @@ -279,6 +290,7 @@ " result = Result(module=module, input=input, conf=conf, time_sec=time)\n", " return result\n", "\n", + "\n", "@torch.inference_mode()\n", "def benchmark(\n", " model,\n", @@ -315,16 +327,25 @@ " ),\n", " ]\n", "\n", - " results = [\n", - " run_configuration_benchmark(deepcopy(model), inputs, conf_)\n", - " for conf_ in configurations\n", - " ]\n", + " results = [run_configuration_benchmark(deepcopy(model), inputs, conf_) for conf_ in configurations]\n", "\n", " for res in results:\n", - " print(res.format())" + " print(res.format())\n", + "\n", + "\n", + "test_model = torchvision.models.resnet18(pretrained=True)\n", + "input = [torch.rand(128, 3, 224, 224)]\n", + "benchmark(test_model, input, 50, 128)" ], - "execution_count": 21, + "execution_count": 8, "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103458.189 manifold.py:1435] URL manifold://torchvision/tree/models/resnet18-f37072fd.pth was already cached in /home/wwei6/.torch/iopath_cache/manifold_cache/tree/models/resnet18-f37072fd.pth\n" + ] + }, { "output_type": "stream", "name": "stdout", @@ -339,25 +360,60 @@ "== End benchmark iterations\n=== Running benchmark for: Configuration(batch_iter=50, batch_size=128, name='TRT FP32 Eager', trt=True, jit=False, fp16=False, accuracy_rtol=0.001) green\n" ] }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103501.297 pass_utils.py:166] == Log pass before/after graph to /tmp/tmpe_7p37fq\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103501.390 pass_utils.py:166] == Log pass before/after graph to /tmp/tmpg_a347f0\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103501.509 lower_pass_manager_builder.py:151] Now lowering submodule _run_on_acc_0\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103501.511 lower.py:89] split_name='_run_on_acc_0' self.lower_setting.input_specs=[InputTensorSpec(shape=torch.Size([128, 3, 224, 224]), dtype=torch.float32, device=device(type='cuda', index=0), shape_ranges=[], has_batch_dim=True)]\n" + ] + }, { "output_type": "stream", "name": "stdout", "text": [ - "== Log pass before/after graph to /tmp/tmpaayayg72\n== Log pass before/after graph to /tmp/tmpdw_pq71j\n\nSupported node types in the model:\nacc_ops.conv2d: ((), {'input': torch.float32, 'weight': torch.float32})\nacc_ops.batch_norm: ((), {'input': torch.float32, 'running_mean': torch.float32, 'running_var': torch.float32, 'weight': torch.float32, 'bias': torch.float32})\nacc_ops.relu: ((), {'input': torch.float32})\nacc_ops.max_pool2d: ((), {'input': torch.float32})\nacc_ops.add: ((), {'input': torch.float32, 'other': torch.float32})\nacc_ops.adaptive_avg_pool2d: ((), {'input': torch.float32})\nacc_ops.flatten: ((), {'input': torch.float32})\nacc_ops.linear: ((), {'input': torch.float32, 'weight': torch.float32, 'bias': torch.float32})\n\nUnsupported node types in the model:\n\nGot 1 acc subgraphs and 0 non-acc subgraphs\n" + "\nSupported node types in the model:\nacc_ops.conv2d: ((), {'input': torch.float32, 'weight': torch.float32})\nacc_ops.batch_norm: ((), {'input': torch.float32, 'running_mean': torch.float32, 'running_var': torch.float32, 'weight': torch.float32, 'bias': torch.float32})\nacc_ops.relu: ((), {'input': torch.float32})\nacc_ops.max_pool2d: ((), {'input': torch.float32})\nacc_ops.add: ((), {'input': torch.float32, 'other': torch.float32})\nacc_ops.adaptive_avg_pool2d: ((), {'input': torch.float32})\nacc_ops.flatten: ((), {'input': torch.float32})\nacc_ops.linear: ((), {'input': torch.float32, 'weight': torch.float32, 'bias': torch.float32})\n\nUnsupported node types in the model:\n\nGot 1 acc subgraphs and 0 non-acc subgraphs\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ - "I0627 233146.650 fx2trt.py:190] Run Module elapsed time: 0:00:00.244369\n" + "I0822 103503.964 fx2trt.py:204] Run Module elapsed time: 0:00:00.435984\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ - "I0627 233206.570 fx2trt.py:241] Build TRT engine elapsed time: 0:00:19.918630\n" + "I0822 103520.647 fx2trt.py:258] Build TRT engine elapsed time: 0:00:16.681226\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103520.658 lower_pass_manager_builder.py:168] Lowering submodule _run_on_acc_0 elapsed time 0:00:19.147071\n" ] }, { @@ -374,25 +430,60 @@ "== End benchmark iterations\n=== Running benchmark for: Configuration(batch_iter=50, batch_size=128, name='TRT FP16 Eager', trt=True, jit=False, fp16=True, accuracy_rtol=0.01) green\n" ] }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103523.067 pass_utils.py:166] == Log pass before/after graph to /tmp/tmpgphlicna\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103523.106 pass_utils.py:166] == Log pass before/after graph to /tmp/tmpy9cumddi\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103523.173 lower_pass_manager_builder.py:151] Now lowering submodule _run_on_acc_0\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103523.174 lower.py:89] split_name='_run_on_acc_0' self.lower_setting.input_specs=[InputTensorSpec(shape=torch.Size([128, 3, 224, 224]), dtype=torch.float16, device=device(type='cuda', index=0), shape_ranges=[], has_batch_dim=True)]\n" + ] + }, { "output_type": "stream", "name": "stdout", "text": [ - "== Log pass before/after graph to /tmp/tmpnoeblgd5\n== Log pass before/after graph to /tmp/tmpyb1egsof\n\nSupported node types in the model:\nacc_ops.conv2d: ((), {'input': torch.float16, 'weight': torch.float16})\nacc_ops.batch_norm: ((), {'input': torch.float16, 'running_mean': torch.float16, 'running_var': torch.float16, 'weight': torch.float16, 'bias': torch.float16})\nacc_ops.relu: ((), {'input': torch.float16})\nacc_ops.max_pool2d: ((), {'input': torch.float16})\nacc_ops.add: ((), {'input': torch.float16, 'other': torch.float16})\nacc_ops.adaptive_avg_pool2d: ((), {'input': torch.float16})\nacc_ops.flatten: ((), {'input': torch.float16})\nacc_ops.linear: ((), {'input': torch.float16, 'weight': torch.float16, 'bias': torch.float16})\n\nUnsupported node types in the model:\n\nGot 1 acc subgraphs and 0 non-acc subgraphs\n" + "\nSupported node types in the model:\nacc_ops.conv2d: ((), {'input': torch.float16, 'weight': torch.float16})\nacc_ops.batch_norm: ((), {'input': torch.float16, 'running_mean': torch.float16, 'running_var': torch.float16, 'weight': torch.float16, 'bias': torch.float16})\nacc_ops.relu: ((), {'input': torch.float16})\nacc_ops.max_pool2d: ((), {'input': torch.float16})\nacc_ops.add: ((), {'input': torch.float16, 'other': torch.float16})\nacc_ops.adaptive_avg_pool2d: ((), {'input': torch.float16})\nacc_ops.flatten: ((), {'input': torch.float16})\nacc_ops.linear: ((), {'input': torch.float16, 'weight': torch.float16, 'bias': torch.float16})\n\nUnsupported node types in the model:\n\nGot 1 acc subgraphs and 0 non-acc subgraphs\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "I0822 103523.466 fx2trt.py:204] Run Module elapsed time: 0:00:00.288043\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ - "I0627 233208.996 fx2trt.py:190] Run Module elapsed time: 0:00:00.217076\n" + "I0822 103553.687 fx2trt.py:258] Build TRT engine elapsed time: 0:00:30.220316\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ - "I0627 233244.147 fx2trt.py:241] Build TRT engine elapsed time: 0:00:35.150950\n" + "I0822 103553.698 lower_pass_manager_builder.py:168] Lowering submodule _run_on_acc_0 elapsed time 0:00:30.523791\n" ] }, { @@ -406,7 +497,7 @@ "output_type": "stream", "name": "stdout", "text": [ - "== End benchmark iterations\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='CUDA Eager', trt=False, jit=False, fp16=False, accuracy_rtol=-1)\nBS: 128, Time per iter: 15.00ms, QPS: 8530.72, Accuracy: None (rtol=-1)\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='TRT FP32 Eager', trt=True, jit=False, fp16=False, accuracy_rtol=0.001)\nBS: 128, Time per iter: 7.95ms, QPS: 16098.45, Accuracy: None (rtol=0.001)\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='TRT FP16 Eager', trt=True, jit=False, fp16=True, accuracy_rtol=0.01)\nBS: 128, Time per iter: 4.36ms, QPS: 29365.31, Accuracy: None (rtol=0.01)\n" + "== End benchmark iterations\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='CUDA Eager', trt=False, jit=False, fp16=False, accuracy_rtol=-1)\nBS: 128, Time per iter: 14.66ms, QPS: 8732.53, Accuracy: None (rtol=-1)\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='TRT FP32 Eager', trt=True, jit=False, fp16=False, accuracy_rtol=0.001)\nBS: 128, Time per iter: 7.27ms, QPS: 17595.70, Accuracy: None (rtol=0.001)\n== Benchmark Result for: Configuration(batch_iter=50, batch_size=128, name='TRT FP16 Eager', trt=True, jit=False, fp16=True, accuracy_rtol=0.01)\nBS: 128, Time per iter: 4.49ms, QPS: 28480.34, Accuracy: None (rtol=0.01)\n" ] } ] From 6e467f23d9007f151d0290ab00fba0eb117ee67b Mon Sep 17 00:00:00 2001 From: Wei Date: Mon, 22 Aug 2022 14:59:14 -0700 Subject: [PATCH 25/31] Changes done internally at Facebook (#1299) bd46e8f292bf68fe6b87d2d5d206c89fda79a746 Shirong Wu Disable group ln fuse pass 6ce1d3bc19d75b266e99355c96daeff7054dcbf8 Wei Wei [fx2trt] set logging level to INFO at fx root 9d552dc3f69db9e4a249f80ef00803a9413e5d38 Wei Wei [fx2trt] change OSS method lower_to_trt() to compile() 6c4bdb8ac5823d161e4afc7c9d295f961aeeb0bf Mor Tzur fix engine holder test binary to fix contbuild_pytorch_fx2trt_build 636d0ab2a3d0f09267e25b8b8e7eedd4d91d791d Yinghai Lu [easy] remove random prints 5a97668307c26e69a89a4e02a535e319eaf3ce3d Wei Wei [ads] sequential linear fuse 508338ab343e407ee49605919508210b62ad9a52 Wei Wei [fx2trt] minor literal fix --- py/torch_tensorrt/fx/__init__.py | 1 - py/torch_tensorrt/fx/lower.py | 2 +- py/torch_tensorrt/fx/passes/pass_utils.py | 47 +++++++++++++++++++ .../fx/tracer/acc_tracer/acc_ops.py | 4 +- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/py/torch_tensorrt/fx/__init__.py b/py/torch_tensorrt/fx/__init__.py index 493d749d09..c1c42c446f 100644 --- a/py/torch_tensorrt/fx/__init__.py +++ b/py/torch_tensorrt/fx/__init__.py @@ -11,6 +11,5 @@ from .input_tensor_spec import generate_input_specs, InputTensorSpec # noqa from .lower_setting import LowerSetting # noqa from .trt_module import TRTModule # noqa -from .lower import compile logging.basicConfig(level=logging.INFO) diff --git a/py/torch_tensorrt/fx/lower.py b/py/torch_tensorrt/fx/lower.py index deeee14178..59b59d580f 100644 --- a/py/torch_tensorrt/fx/lower.py +++ b/py/torch_tensorrt/fx/lower.py @@ -53,7 +53,7 @@ def compile( timing_cache_prefix: Timing cache file name for timing cache used by fx2trt. save_timing_cache: Update timing cache with current timing cache data if set to True. cuda_graph_batch_size: Cuda graph batch size, default to be -1. - + dynamic_batch: batch dimension (dim=0) is dynamic. Returns: A torch.nn.Module lowered by TensorRT. """ diff --git a/py/torch_tensorrt/fx/passes/pass_utils.py b/py/torch_tensorrt/fx/passes/pass_utils.py index d430a67408..3fb88e04a9 100644 --- a/py/torch_tensorrt/fx/passes/pass_utils.py +++ b/py/torch_tensorrt/fx/passes/pass_utils.py @@ -102,6 +102,53 @@ def bounded_method(*args, **kwargs): return dec_for_method +def log_perf_before_after(pass_: PassFunc) -> PassFunc: + """ + Wraps a pass function to log perf of the module before and after the pass + """ + + @wraps(pass_) + def check_perf_with_before_after_log( + module: fx.GraphModule, input: Input + ) -> fx.GraphModule: + def benchmark_torch_function(iters: int, f, *args) -> float: + """Estimates the average time duration for a single inference call in second + + If the input is batched, then the estimation is for the batches inference call. + + Args: + iters: number of inference iterations to run + f: a function to perform a single inference call + + Returns: + estimated average time duration in second for a single inference call + """ + with torch.inference_mode(): + f(*args) + torch.cuda.synchronize() + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + # print("== Start benchmark iterations") + with torch.inference_mode(): + start_event.record() + for _ in range(iters): + f(*args) + end_event.record() + torch.cuda.synchronize() + # print("== End benchmark iterations") + return (start_event.elapsed_time(end_event) * 1.0e-3) / iters + + time_before = benchmark_torch_function(100, lambda: module(*input)) + _LOGGER.info(f"[{pass_}] Perf Before(eager mode): {time_before}") + + module = pass_(module, input) + time_after = benchmark_torch_function(100, lambda: module(*input)) + _LOGGER.info(f"[{pass_}] Perf After(eager mode): {time_after}") + return module + + return check_perf_with_before_after_log + + def log_before_after(pass_: PassFunc) -> PassFunc: """ Wraps a pass function to log the module graph before and after the pass diff --git a/py/torch_tensorrt/fx/tracer/acc_tracer/acc_ops.py b/py/torch_tensorrt/fx/tracer/acc_tracer/acc_ops.py index d1a5322316..ccd572b9aa 100644 --- a/py/torch_tensorrt/fx/tracer/acc_tracer/acc_ops.py +++ b/py/torch_tensorrt/fx/tracer/acc_tracer/acc_ops.py @@ -778,7 +778,9 @@ def dropout_mapper(node: torch.fx.Node, mod: nn.Module): assert callable(stochastic_depth) except Exception as e: - warnings.warn(f"Unable to import torchvision related libraries.: {e}") + warnings.warn( + f"Unable to import torchvision related libraries.: {e}. Please install torchvision lib in order to lower stochastic_depth" + ) else: @register_custom_acc_mapper_fn( From fbdffdcf8789efd7d9f62757be84a83b4a7896b4 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Mon, 22 Aug 2022 15:00:27 -0700 Subject: [PATCH 26/31] Update docgen.yml --- .github/workflows/docgen.yml | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 8a3fd24717..2a4310d622 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -28,16 +28,10 @@ jobs: apt update DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common gcc git curl wget make zlib1g-dev bzip2 libbz2-dev lzma lzma-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev doxygen pandoc git config --global --add safe.directory '*' - - name: Install Python - run: | - mkdir -p /opt/circleci - git clone https://github.com/pyenv/pyenv.git /opt/circleci/.pyenv - export PYENV_ROOT="/opt/circleci/.pyenv" - export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" - pyenv install 3.9.4 - pyenv global 3.9.4 - python3 -m pip install --upgrade pip - python3 -m pip install wheel + - name: Set up Python 3.9.4 + uses: actions/setup-python@v2 + with: + python-version: 3.9.4 - uses: actions/checkout@v2 with: ref: ${{github.head_ref}} @@ -70,25 +64,19 @@ jobs: apt-get install -y libnvinfer8 libnvinfer-plugin8 libnvinfer-dev libnvinfer-plugin-dev - name: Install Torch run: | - export PYENV_ROOT="/opt/circleci/.pyenv" - export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" - pyenv global 3.9.4 python3 -m pip install -r py/requirements.txt - name: Build Python Package run: | cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py - export PYENV_ROOT="/opt/circleci/.pyenv" - export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" - pyenv global 3.9.4 + echo $(which python3) + mkdir -p /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/ + ln -s /usr/local/lib/python3.8/site-packages /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages python3 setup.py install cd .. - name: Generate New Docs run: | cd docsrc - export PYENV_ROOT="/opt/circleci/.pyenv" - export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" - pyenv global 3.9.4 python3 -m pip install -r requirements.txt python3 -c "import torch_tensorrt; print(torch_tensorrt.__version__)" make html From 399c763f74533bbccfca3a7861817783c1f626ba Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Mon, 22 Aug 2022 15:52:31 -0700 Subject: [PATCH 27/31] Update docgen.yml --- .github/workflows/docgen.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 2a4310d622..28d0752328 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -70,8 +70,9 @@ jobs: cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py echo $(which python3) + ls $(which python3)/.. mkdir -p /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/ - ln -s /usr/local/lib/python3.8/site-packages /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages + ln -s /__t/Python/3.9.4/x64/lib/python3.9/site-packages /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages python3 setup.py install cd .. - name: Generate New Docs From d2a81436d8490135cd022776fca999c4f8a126b8 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Mon, 22 Aug 2022 16:01:58 -0700 Subject: [PATCH 28/31] Update docgen.yml --- .github/workflows/docgen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 28d0752328..79b9c74124 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -70,9 +70,9 @@ jobs: cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py echo $(which python3) - ls $(which python3)/.. + echo $(python3 -c 'import site; print(site.getsitepackages())') mkdir -p /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/ - ln -s /__t/Python/3.9.4/x64/lib/python3.9/site-packages /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages + ln -s $(python3 -c 'import site; print(site.getsitepackages())') /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages python3 setup.py install cd .. - name: Generate New Docs From 0baf50abb128de50afe827b783ae1b542d161dff Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Mon, 22 Aug 2022 16:11:07 -0700 Subject: [PATCH 29/31] Update docgen.yml --- .github/workflows/docgen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 79b9c74124..7b66b98be5 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -70,9 +70,9 @@ jobs: cp toolchains/ci_workspaces/WORKSPACE.x86_64 WORKSPACE cd py echo $(which python3) - echo $(python3 -c 'import site; print(site.getsitepackages())') + echo $(python3 -c 'import site; print(site.getsitepackages()[0])') mkdir -p /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/ - ln -s $(python3 -c 'import site; print(site.getsitepackages())') /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages + ln -s $(python3 -c 'import site; print(site.getsitepackages()[0])') /opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages python3 setup.py install cd .. - name: Generate New Docs From 0005137d42d4603ecd9a57d6d0071ac21520273d Mon Sep 17 00:00:00 2001 From: Torch-TensorRT Github Bot Date: Mon, 22 Aug 2022 23:38:33 +0000 Subject: [PATCH 30/31] docs: [Automated] Regenerating documenation for 0baf50a Signed-off-by: Torch-TensorRT Github Bot --- .../classtorch__tensorrt_1_1DataType.html | 172 ++- ...rch__tensorrt_1_1Device_1_1DeviceType.html | 170 ++- .../classtorch__tensorrt_1_1TensorFormat.html | 172 ++- ...ensorrt_1_1ptq_1_1Int8CacheCalibrator.html | 170 ++- ...ch__tensorrt_1_1ptq_1_1Int8Calibrator.html | 170 ++- ...8h_1a18d295a837ac71add5578860b55e5502.html | 170 ++- ...8h_1a282fd3c0b1c3a215148ae372070e1268.html | 170 ++- ...8h_1a31398a6d4d27e28817afb0f0139e909e.html | 170 ++- ...8h_1a35703561b26b1a9d2738ad7d58b27827.html | 170 ++- ...8h_1abd1465eb38256d3f22cc1426b23d516b.html | 170 ++- ...8h_1abe87b341f562fd1cf40b7672e4d759da.html | 174 ++- ...8h_1ad19939408f7be171a74a89928b36eb59.html | 170 ++- ...8h_1adad592a7b1b7eed529cdf6acd584c883.html | 170 ++- docs/_cpp_api/dir_cpp.html | 158 ++- docs/_cpp_api/dir_cpp_include.html | 158 ++- .../dir_cpp_include_torch_tensorrt.html | 158 ++- ...8h_1a130f65408ad8cbaee060f05e8db69558.html | 170 ++- ...8h_1a3fbe5d72e4fc624dbd038853079620eb.html | 170 ++- ..._cpp_include_torch_tensorrt_logging.h.html | 158 ++- ...e_cpp_include_torch_tensorrt_macros.h.html | 159 ++- ...file_cpp_include_torch_tensorrt_ptq.h.html | 163 ++- ...clude_torch_tensorrt_torch_tensorrt.h.html | 160 +-- ...8h_1a0593f776f469c20469e2f729fc7861a3.html | 170 ++- ...8h_1a0c012cb374addd90eb1f42eaec570650.html | 170 ++- ...8h_1a56e110feaaba2c3fd44bd201fd21a76a.html | 170 ++- ...8h_1a7cb50492421ea9de4e3db895819df6f2.html | 174 ++- ...8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.html | 170 ++- ...8h_1ad2efd47b6c3689e58ccc595680579ae5.html | 170 ++- ...8h_1af8f3443813315af7901903d25dd495cc.html | 170 ++- ...8h_1a226e3c83379d1012cde8578c1c86b16c.html | 674 +++++++++++ ...8h_1a6186e305f47c1d94b6130ef6c7f7e178.html | 680 +++++++++++ ...8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.html | 170 ++- ...8h_1a6e19490a08fb1553c9dd347a5ae79db9.html | 170 ++- ...8h_1a710df824a7718b440e4bc17bf9693cef.html | 170 ++- ...8h_1ac4ab8313ae72c2c899ea31548b528528.html | 170 ++- ...8h_1ad1acd06eaeaffbbcf6e7ebf426891384.html | 174 ++- ...8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.html | 170 ++- ...8h_1ae8d56472106eeef37fbe51ff7f40c9b2.html | 170 ++- docs/_cpp_api/namespace_torch_tensorrt.html | 171 ++- .../namespace_torch_tensorrt__logging.html | 170 ++- .../namespace_torch_tensorrt__ptq.html | 174 ++- ...namespace_torch_tensorrt__torchscript.html | 170 ++- ..._cpp_include_torch_tensorrt_logging.h.html | 158 ++- ...e_cpp_include_torch_tensorrt_macros.h.html | 162 +-- ...file_cpp_include_torch_tensorrt_ptq.h.html | 165 ++- ...clude_torch_tensorrt_torch_tensorrt.h.html | 221 ++-- .../structtorch__tensorrt_1_1Device.html | 174 ++- .../structtorch__tensorrt_1_1GraphInputs.html | 674 +++++++++++ .../structtorch__tensorrt_1_1Input.html | 212 ++-- ...ensorrt_1_1torchscript_1_1CompileSpec.html | 222 ++-- docs/_cpp_api/torch_tensort_cpp.html | 194 ++-- docs/_cpp_api/unabridged_orphan.html | 158 ++- docs/_modules/index.html | 159 +-- docs/_modules/torch_tensorrt/_Device.html | 197 ++-- docs/_modules/torch_tensorrt/_Input.html | 259 +++-- docs/_modules/torch_tensorrt/_compile.html | 324 +++--- docs/_modules/torch_tensorrt/_util.html | 173 +-- docs/_modules/torch_tensorrt/fx/fx2trt.html | 999 ++++++++++++++++ .../torch_tensorrt/fx/input_tensor_spec.html | 809 +++++++++++++ .../torch_tensorrt/fx/trt_module.html | 863 ++++++++++++++ docs/_modules/torch_tensorrt/logging.html | 176 +-- docs/_modules/torch_tensorrt/ptq.html | 260 +++-- .../torch_tensorrt/ts/_compile_spec.html | 424 ++++--- .../_modules/torch_tensorrt/ts/_compiler.html | 306 ++--- ...pp_include_torch_tensorrt_macros.h.rst.txt | 2 + ...e_cpp_include_torch_tensorrt_ptq.h.rst.txt | 6 +- ...de_torch_tensorrt_torch_tensorrt.h.rst.txt | 4 + ...1a226e3c83379d1012cde8578c1c86b16c.rst.txt | 14 + ...1a6186e305f47c1d94b6130ef6c7f7e178.rst.txt | 14 + .../_cpp_api/namespace_torch_tensorrt.rst.txt | 2 + .../namespace_torch_tensorrt__ptq.rst.txt | 4 +- ...p_include_torch_tensorrt_logging.h.rst.txt | 16 +- ...pp_include_torch_tensorrt_macros.h.rst.txt | 12 +- ...e_cpp_include_torch_tensorrt_ptq.h.rst.txt | 57 +- ...de_torch_tensorrt_torch_tensorrt.h.rst.txt | 177 +-- ...ructtorch__tensorrt_1_1GraphInputs.rst.txt | 17 + .../structtorch__tensorrt_1_1Input.rst.txt | 9 + docs/_sources/cli/torchtrtc.rst.txt | 146 +++ .../contributors/partitioning.rst.txt | 2 +- .../getting_started_with_cpp_api.rst.txt | 338 ++++++ .../getting_started_with_python_api.rst.txt | 58 + .../getting_started/installation.rst.txt | 362 ++++++ docs/_sources/index.rst.txt | 65 +- docs/_sources/py_api/fx.rst.txt | 31 + docs/_sources/py_api/ptq.rst.txt | 2 +- docs/_sources/py_api/torch_tensorrt.rst.txt | 2 +- .../docs/demo/demo.rst.txt | 4 +- .../docs/demo/lists_tables.rst.txt | 2 +- .../docs/demo/structure.rst.txt | 8 +- ...ating_torchscript_module_in_python.rst.txt | 7 +- .../getting_started_with_fx_path.rst.txt | 297 +++++ docs/_sources/tutorials/notebooks.rst.txt | 154 +++ docs/_sources/tutorials/ptq.rst.txt | 77 +- .../tutorials/use_from_pytorch.rst.txt | 12 +- docs/_sources/tutorials/using_dla.rst.txt | 5 +- docs/_static/css/theme.css | 26 + docs/cli/torchtrtc.html | 776 +++++++++++++ docs/contributors/conversion.html | 170 ++- docs/contributors/lowering.html | 170 ++- docs/contributors/partitioning.html | 170 ++- docs/contributors/phases.html | 158 ++- docs/contributors/runtime.html | 170 ++- docs/contributors/system_overview.html | 172 ++- docs/contributors/useful_links.html | 168 ++- docs/contributors/writing_converters.html | 168 ++- docs/genindex.html | 205 ++-- .../getting_started_with_cpp_api.html | 935 +++++++++++++++ .../getting_started_with_python_api.html | 689 +++++++++++ docs/getting_started/installation.html | 1012 +++++++++++++++++ docs/index.html | 207 ++-- docs/indices/supported_ops.html | 166 ++- docs/objects.inv | Bin 24856 -> 24827 bytes docs/py-modindex.html | 161 +-- docs/py_api/fx.html | 691 +++++++++++ docs/py_api/logging.html | 170 ++- docs/py_api/ptq.html | 170 ++- docs/py_api/torch_tensorrt.html | 173 ++- docs/py_api/ts.html | 219 ++-- docs/search.html | 164 ++- docs/searchindex.js | 2 +- .../pytorch-sphinx-theme/docs/changelog.html | 158 ++- .../docs/configuring.html | 158 ++- .../pytorch-sphinx-theme/docs/demo/api.html | 158 ++- .../pytorch-sphinx-theme/docs/demo/demo.html | 160 ++- .../docs/demo/lists_tables.html | 158 ++- .../pytorch-sphinx-theme/docs/demo/long.html | 158 ++- .../docs/demo/structure.html | 158 ++- docs/src/pytorch-sphinx-theme/docs/index.html | 158 ++- .../pytorch-sphinx-theme/docs/installing.html | 158 ++- ...creating_torchscript_module_in_python.html | 251 ++-- .../getting_started_with_fx_path.html | 912 +++++++++++++++ docs/tutorials/notebooks.html | 795 +++++++++++++ docs/tutorials/ptq.html | 253 ++--- docs/tutorials/runtime.html | 178 ++- .../serving_torch_tensorrt_with_triton.html | 176 ++- docs/tutorials/use_from_pytorch.html | 190 ++-- docs/tutorials/using_dla.html | 181 ++- 137 files changed, 20507 insertions(+), 8259 deletions(-) create mode 100644 docs/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.html create mode 100644 docs/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.html create mode 100644 docs/_cpp_api/structtorch__tensorrt_1_1GraphInputs.html create mode 100644 docs/_modules/torch_tensorrt/fx/fx2trt.html create mode 100644 docs/_modules/torch_tensorrt/fx/input_tensor_spec.html create mode 100644 docs/_modules/torch_tensorrt/fx/trt_module.html create mode 100644 docs/_sources/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.rst.txt create mode 100644 docs/_sources/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.rst.txt create mode 100644 docs/_sources/_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst.txt create mode 100644 docs/_sources/cli/torchtrtc.rst.txt create mode 100644 docs/_sources/getting_started/getting_started_with_cpp_api.rst.txt create mode 100644 docs/_sources/getting_started/getting_started_with_python_api.rst.txt create mode 100644 docs/_sources/getting_started/installation.rst.txt create mode 100644 docs/_sources/py_api/fx.rst.txt create mode 100644 docs/_sources/tutorials/getting_started_with_fx_path.rst.txt create mode 100644 docs/_sources/tutorials/notebooks.rst.txt create mode 100644 docs/cli/torchtrtc.html create mode 100644 docs/getting_started/getting_started_with_cpp_api.html create mode 100644 docs/getting_started/getting_started_with_python_api.html create mode 100644 docs/getting_started/installation.html create mode 100644 docs/py_api/fx.html create mode 100644 docs/tutorials/getting_started_with_fx_path.html create mode 100644 docs/tutorials/notebooks.html diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html b/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html index 64a3a981be..222e3ae811 100644 --- a/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html +++ b/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html @@ -9,22 +9,22 @@ - + Class DataType — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Class DataType

+ - - +
- +

@@ -554,11 +552,11 @@

Class DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -581,12 +579,12 @@

Class Documentation @@ -595,9 +593,9 @@

Class Documentation + - - + @@ -608,7 +606,7 @@

Class Documentation - + Class Device::DeviceType — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
+ - - +
- +

@@ -502,11 +500,11 @@

Class DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -530,12 +528,12 @@

Class Documentation @@ -544,9 +542,9 @@

Class Documentation + - - + @@ -557,7 +555,7 @@

Class Documentation - + Class TensorFormat — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
+ - - +
- +

@@ -535,11 +533,11 @@

Class DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -562,12 +560,12 @@

Class Documentation @@ -576,9 +574,9 @@

Class Documentation + - - + @@ -589,7 +587,7 @@

Class Documentation - + Template Class Int8CacheCalibrator — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Template Class Int8CacheCalibrator

+ - - +
- +

@@ -495,11 +493,11 @@

Class DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -526,12 +524,12 @@

Class Documentation @@ -540,9 +538,9 @@

Class Documentation + - - + @@ -553,7 +551,7 @@

Class Documentation - + Template Class Int8Calibrator — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Template Class Int8Calibrator

+ - - +
- +

@@ -505,11 +503,11 @@

Class DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -536,12 +534,12 @@

Class Documentation @@ -550,9 +548,9 @@

Class Documentation + - - + @@ -563,7 +561,7 @@

Class Documentation - + Define STR — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define STR

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html b/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html index ed4dfb1e18..270688bdf0 100644 --- a/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html +++ b/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html @@ -9,22 +9,22 @@ - + Define TORCH_TENSORRT_PATCH_VERSION — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCH_TENSORRT_PATCH_VERSION

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html b/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html index 6aefafdac6..8315d96c12 100644 --- a/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html +++ b/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html @@ -9,22 +9,22 @@ - + Define TORCH_TENSORRT_MAJOR_VERSION — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCH_TENSORRT_MAJOR_VERSION

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html b/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html index e4450171f4..55d141debe 100644 --- a/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html +++ b/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html @@ -9,22 +9,22 @@ - + Define TORCH_TENSORRT_MINOR_VERSION — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCH_TENSORRT_MINOR_VERSION

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html b/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html index f1a14e6557..f23fdf91d9 100644 --- a/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html +++ b/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html @@ -9,22 +9,22 @@ - + Define TORCHTRT_API — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCHTRT_API

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html b/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html index d3c81be46a..181d018bc0 100644 --- a/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html +++ b/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html @@ -9,22 +9,22 @@ - + Define XSTR — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define XSTR

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html b/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html index da099f0a00..ee7244b879 100644 --- a/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html +++ b/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html @@ -9,22 +9,22 @@ - + Define TORCHTRT_HIDDEN — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCHTRT_HIDDEN

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html b/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html index d06284d4e7..0ba7bd6ec6 100644 --- a/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html +++ b/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html @@ -9,22 +9,22 @@ - + Define TORCH_TENSORRT_VERSION — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
- +

Define TORCH_TENSORRT_VERSION

    @@ -368,25 +366,25 @@

    Define Documentation - +

@@ -421,12 +419,12 @@

Define Documentation

+ + - - - + @@ -435,9 +433,9 @@

Define Documentation + - - + @@ -448,7 +446,7 @@

Define Documentation + diff --git a/docs/_cpp_api/dir_cpp.html b/docs/_cpp_api/dir_cpp.html index db15cf55e8..dadc843051 100644 --- a/docs/_cpp_api/dir_cpp.html +++ b/docs/_cpp_api/dir_cpp.html @@ -9,22 +9,22 @@ - + Directory cpp — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
- +
@@ -340,13 +338,13 @@
- - + +
- +
- + @@ -404,12 +402,12 @@

Subdirectories @@ -418,9 +416,9 @@

Subdirectories + - - + @@ -431,7 +429,7 @@

Subdirectories - + Directory include — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
    - +
  • - + Docs - + >
    • - +
  • Directory include
    • - - + +
  • - - + + - - + +
    • - +
- +
@@ -340,13 +338,13 @@
- - + +
- +
- + @@ -405,12 +403,12 @@

Subdirectories @@ -419,9 +417,9 @@

Subdirectories + - - + @@ -432,7 +430,7 @@

Subdirectories - + Directory torch_tensorrt — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
    - +
  • - + Docs - + >
    • - +
  • Directory torch_tensorrt
    • - - + +
  • - - + + - - + +
    • - +
- +
@@ -340,13 +338,13 @@
- - + +
- +
- +

Directory torch_tensorrt

↰ Parent directory (cpp/include)

@@ -364,16 +362,16 @@

Files

- +
@@ -408,12 +406,12 @@

Files

+ + - - - + @@ -422,9 +420,9 @@

Files + - - + @@ -435,7 +433,7 @@

Files jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.html b/docs/_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.html index 4101739220..bfac604282 100644 --- a/docs/_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.html +++ b/docs/_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.html @@ -9,22 +9,22 @@ - + Enum Level — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
+ - - +
- +

@@ -432,11 +430,11 @@

Enum DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -459,12 +457,12 @@

Enum Documentation @@ -473,9 +471,9 @@

Enum Documentation + - - + @@ -486,7 +484,7 @@

Enum Documentation - + Enum EngineCapability — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
Table of Contents @@ -193,21 +193,21 @@
@@ -288,7 +286,7 @@
- + @@ -307,32 +305,32 @@
- +
@@ -344,13 +342,13 @@
- - + +
- +
+ - - +
- +

@@ -411,11 +409,11 @@

Enum DocumentationSphinx using a theme provided by Read the Docs.

- + @@ -438,12 +436,12 @@

Enum Documentation @@ -452,9 +450,9 @@

Enum Documentation + - - + @@ -465,7 +463,7 @@

Enum Documentation - + File logging.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
- +
@@ -340,13 +338,13 @@
- - + +
- +
- + @@ -460,12 +458,12 @@

Functions @@ -474,9 +472,9 @@

Functions + - - + @@ -487,7 +485,7 @@

Functions - + File macros.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
- +
@@ -340,13 +338,13 @@
- - + +
- +
- + @@ -445,12 +444,12 @@

Defines @@ -459,9 +458,9 @@

Defines + - - + @@ -472,7 +471,7 @@

Defines - + File ptq.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
Table of Contents @@ -191,21 +191,21 @@
@@ -286,7 +284,7 @@
- + @@ -305,30 +303,30 @@
- +
@@ -340,13 +338,13 @@
- - + +
- +
- +

File ptq.h

↰ Parent directory (cpp/include/torch_tensorrt)

@@ -380,6 +378,7 @@

Includesstring

  • torch/torch.h

  • torch_tensorrt/logging.h (File logging.h)

    • +

  • torch_tensorrt/macros.h (File macros.h)

  • vector

    • @@ -400,24 +399,24 @@

    Classes

    Functions

    - +
    @@ -456,12 +455,12 @@

    Functions @@ -470,9 +469,9 @@

    Functions + - - + @@ -483,7 +482,7 @@

    Functions - + File torch_tensorrt.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • File torch_tensorrt.h
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - + @@ -470,12 +470,12 @@

    Functions @@ -484,9 +484,9 @@

    Functions + - - + @@ -497,7 +497,7 @@

    Functions - + Function torch_tensorrt::logging::get_logging_prefix — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::get_logging_prefix

      @@ -368,25 +366,25 @@

      Function Documentation - +

    @@ -421,12 +419,12 @@

    Function Documentation

    + + - - - + @@ -435,9 +433,9 @@

    Function Documentation + - - + @@ -448,7 +446,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.html b/docs/_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.html index 9ffa0ccb73..d98c9041c6 100644 --- a/docs/_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.html +++ b/docs/_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::get_reportable_log_level — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::get_reportable_log_level

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.html b/docs/_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.html index e2936c2415..1c35f0615a 100644 --- a/docs/_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.html +++ b/docs/_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::get_is_colored_output_on — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::get_is_colored_output_on

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.html b/docs/_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.html index 16cddb8436..e18c749cbd 100644 --- a/docs/_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.html +++ b/docs/_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::set_reportable_log_level — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::set_reportable_log_level

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.html b/docs/_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.html index fc0a366d18..833851e934 100644 --- a/docs/_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.html +++ b/docs/_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::log — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::log

      @@ -377,25 +375,25 @@

      Function Documentation - +

    @@ -430,12 +428,12 @@

    Function Documentation

    + + - - - + @@ -444,9 +442,9 @@

    Function Documentation + - - + @@ -457,7 +455,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.html b/docs/_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.html index 20fd61ccfb..2682bb0385 100644 --- a/docs/_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.html +++ b/docs/_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::set_is_colored_output_on — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::set_is_colored_output_on

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.html b/docs/_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.html index a6705d73c8..0cf0d8f848 100644 --- a/docs/_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.html +++ b/docs/_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::logging::set_logging_prefix — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::logging::set_logging_prefix

      @@ -368,25 +366,25 @@

      Function Documentation - +

    @@ -421,12 +419,12 @@

    Function Documentation

    + + - - - + @@ -435,9 +433,9 @@

    Function Documentation + - - + @@ -448,7 +446,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.html b/docs/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.html new file mode 100644 index 0000000000..e7ea78f4fd --- /dev/null +++ b/docs/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.html @@ -0,0 +1,674 @@ + + + + + + + + + + + + + Template Function torch_tensorrt::ptq::make_int8_cache_calibrator — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + + + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Template Function torch_tensorrt::ptq::make_int8_cache_calibrator

    + +
    +

    Function Documentation

    +
    +
    +template<typename Algorithm = nvinfer1::IInt8EntropyCalibrator2>
    inline Int8CacheCalibrator<Algorithm> torch_tensorrt::ptq::make_int8_cache_calibrator(const std::string &cache_file_path)
    +

    A factory to build a post training quantization calibrator from a torch dataloader that only uses the calibration cache.

    +

    Creates a calibrator to use for post training quantization which reads from a previously created calibration cache, therefore you can have a calibration cache generating program that requires a dataloader and a dataset, then save the cache to use later in a different program that needs to calibrate from scratch and not have the dataset dependency. However, the network should also be recalibrated if its structure changes, or the input data set changes, and it is the responsibility of the application to ensure this.

    +

    By default the returned calibrator uses TensorRT Entropy v2 algorithm to perform calibration. This is recommended for feed forward networks You can override the algorithm selection (such as to use the MinMax Calibrator recomended for NLP tasks) by calling make_int8_calibrator with the calibrator class as a template parameter.

    +

    e.g. torch_tensorrt::ptq::make_int8_cache_calibrator<nvinfer1::IInt8MinMaxCalibrator>(calibration_cache_file);

    +
    +
    Template Parameters
    +

    Algorithm – class nvinfer1::IInt8Calibrator (Default: nvinfer1::IInt8EntropyCalibrator2) - Algorithm to use

    +
    +
    Parameters
    +

    cache_file_path – const std::string& - Path to read/write calibration cache

    +
    +
    Returns
    +

    Int8CacheCalibrator<Algorithm>

    +
    +
    +
    + +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.html b/docs/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.html new file mode 100644 index 0000000000..12b173be82 --- /dev/null +++ b/docs/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.html @@ -0,0 +1,680 @@ + + + + + + + + + + + + + Template Function torch_tensorrt::ptq::make_int8_calibrator — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + + + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Template Function torch_tensorrt::ptq::make_int8_calibrator

    + +
    +

    Function Documentation

    +
    +
    +template<typename Algorithm = nvinfer1::IInt8EntropyCalibrator2, typename DataLoader>
    inline Int8Calibrator<Algorithm, DataLoader> torch_tensorrt::ptq::make_int8_calibrator(DataLoader dataloader, const std::string &cache_file_path, bool use_cache)
    +

    A factory to build a post training quantization calibrator from a torch dataloader.

    +

    Creates a calibrator to use for post training quantization. By default the returned calibrator uses TensorRT Entropy v2 algorithm to perform calibration. This is recommended for feed forward networks. You can override the algorithm selection (such as to use the MinMax Calibrator recomended for NLP tasks) by calling make_int8_calibrator with the calibrator class as a template parameter.

    +

    e.g. torch_tensorrt::ptq::make_int8_calibrator<nvinfer1::IInt8MinMaxCalibrator>(std::move(calibration_dataloader), calibration_cache_file, use_cache);

    +
    +
    Template Parameters
    +
      +

    • Algorithm – class nvinfer1::IInt8Calibrator (Default: nvinfer1::IInt8EntropyCalibrator2) - Algorithm to use

      • +

    • DataLoader – std::unique_ptr<torch::data::DataLoader> - DataLoader type

      • +
    +
    +
    Parameters
    +
      +

    • dataloader – std::unique_ptr<torch::data::DataLoader> - DataLoader containing data

      • +

    • cache_file_path – const std::string& - Path to read/write calibration cache

      • +

    • use_cache – bool - use calibration cache

      • +
    +
    +
    Returns
    +

    Int8Calibrator<Algorithm, DataLoader>

    +
    +
    +
    + +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.html b/docs/_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.html index 832606f12f..efdad0ff1e 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::torchscript::check_method_operator_support — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::torchscript::check_method_operator_support

      @@ -383,25 +381,25 @@

      Function Documentation - +

    @@ -436,12 +434,12 @@

    Function Documentation

    + + - - - + @@ -450,9 +448,9 @@

    Function Documentation + - - + @@ -463,7 +461,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.html b/docs/_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.html index 1c36a08a21..2577c6b6ac 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::torchscript::compile — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::torchscript::compile

      @@ -383,25 +381,25 @@

      Function Documentation - +

    @@ -436,12 +434,12 @@

    Function Documentation

    + + - - - + @@ -450,9 +448,9 @@

    Function Documentation + - - + @@ -463,7 +461,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.html b/docs/_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.html index 4dc28fa938..cfeed3ed2d 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::torchscript::embed_engine_in_new_module — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::torchscript::embed_engine_in_new_module

      @@ -387,25 +385,25 @@

      Function Documentation - +

    @@ -440,12 +438,12 @@

    Function Documentation

    + + - - - + @@ -454,9 +452,9 @@

    Function Documentation + - - + @@ -467,7 +465,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.html b/docs/_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.html index afbb934852..04c604233b 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::get_build_info — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::get_build_info

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.html b/docs/_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.html index c4d193a3fa..25dd189a5f 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::set_device — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,13 +34,13 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::set_device

      @@ -374,25 +372,25 @@

      Function Documentation - +

    @@ -427,12 +425,12 @@

    Function Documentation

    + + - - - + @@ -441,9 +439,9 @@

    Function Documentation + - - + @@ -454,7 +452,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.html b/docs/_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.html index 65ea8ab9ab..c57c82027c 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::dump_build_info — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::dump_build_info

      @@ -369,25 +367,25 @@

      Function Documentation - +

    @@ -422,12 +420,12 @@

    Function Documentation

    + + - - - + @@ -436,9 +434,9 @@

    Function Documentation + - - + @@ -449,7 +447,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.html b/docs/_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.html index 34d56fce45..c6ad139812 100644 --- a/docs/_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.html +++ b/docs/_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.html @@ -9,22 +9,22 @@ - + Function torch_tensorrt::torchscript::convert_method_to_trt_engine — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Function torch_tensorrt::torchscript::convert_method_to_trt_engine

      @@ -383,25 +381,25 @@

      Function Documentation - +

    @@ -436,12 +434,12 @@

    Function Documentation

    + + - - - + @@ -450,9 +448,9 @@

    Function Documentation + - - + @@ -463,7 +461,7 @@

    Function Documentation + diff --git a/docs/_cpp_api/namespace_torch_tensorrt.html b/docs/_cpp_api/namespace_torch_tensorrt.html index 81c6aa2ecf..53dfca2515 100644 --- a/docs/_cpp_api/namespace_torch_tensorrt.html +++ b/docs/_cpp_api/namespace_torch_tensorrt.html @@ -9,22 +9,22 @@ - + Namespace torch_tensorrt — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Namespace torch_tensorrt

    @@ -374,6 +372,7 @@

    Namespaces

    + - - +
    - +

    @@ -424,11 +423,11 @@

    FunctionsSphinx using a theme provided by Read the Docs.

    - + @@ -454,12 +453,12 @@

    Functions @@ -468,9 +467,9 @@

    Functions + - - + @@ -481,7 +480,7 @@

    Functions - + Namespace torch_tensorrt::logging — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Namespace torch_tensorrt::logging

    @@ -382,25 +380,25 @@

    Functions - + - - + + - +

    + - - +
    - +

    @@ -408,11 +406,11 @@

    FunctionsSphinx using a theme provided by Read the Docs.

    - + @@ -436,12 +434,12 @@

    Functions @@ -450,9 +448,9 @@

    Functions + - - + @@ -463,7 +461,7 @@

    Functions - + Namespace torch_tensorrt::ptq — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +

    - +
    @@ -432,12 +430,12 @@

    Functions @@ -446,9 +444,9 @@

    Functions + - - + @@ -459,7 +457,7 @@

    Functions - + Namespace torch_tensorrt::torchscript — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Namespace torch_tensorrt::torchscript

    @@ -379,25 +377,25 @@

    Functions - + - - + + - +

    + - - +
    - +

    @@ -405,11 +403,11 @@

    FunctionsSphinx using a theme provided by Read the Docs.

    - + @@ -433,12 +431,12 @@

    Functions @@ -447,9 +445,9 @@

    Functions + - - + @@ -460,7 +458,7 @@

    Functions - + Program Listing for File logging.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • Program Listing for File logging.h
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +
    @@ -436,12 +434,12 @@
    + + - - - + @@ -450,9 +448,9 @@ + - - + @@ -463,7 +461,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.html b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.html index 85e21b7c86..34a9e2bab4 100644 --- a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.html +++ b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.html @@ -9,22 +9,22 @@ - + Program Listing for File macros.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • Program Listing for File macros.h
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +

    Program Listing for File macros.h

    ↰ Return to documentation for file (cpp/include/torch_tensorrt/macros.h)

    @@ -359,6 +357,9 @@ */ #pragma once +#if defined(USE_CMAKE_GENERATED_EXPORT_HEADER) +#include <torch_tensorrt_export.h> +#else #if defined(__GNUC__) #define TORCHTRT_API __attribute__((__visibility__("default"))) #define TORCHTRT_HIDDEN __attribute__((__visibility__("hidden"))) @@ -366,6 +367,7 @@ #define TORCHTRT_API #define TORCHTRT_HIDDEN #endif // defined(__GNUC__) +#endif // defined(USE_CMAKE_GENERATED_EXPORT_HEADER) // Does this need to be gaurded or something? #define XSTR(x) #x @@ -390,16 +392,16 @@
    - +
    @@ -431,12 +433,12 @@
    + + - - - + @@ -445,9 +447,9 @@ + - - + @@ -458,7 +460,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.html b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.html index aa944efd78..bb9f32495e 100644 --- a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.html +++ b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.html @@ -9,22 +9,22 @@ - + Program Listing for File ptq.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • Program Listing for File ptq.h
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +

    Program Listing for File ptq.h

    ↰ Return to documentation for file (cpp/include/torch_tensorrt/ptq.h)

    @@ -370,6 +368,7 @@ #include "NvInfer.h" #include "torch/torch.h" #include "torch_tensorrt/logging.h" +#include "torch_tensorrt/macros.h" #ifndef DOXYGEN_SHOULD_SKIP_THIS namespace nvinfer1 { @@ -379,7 +378,7 @@ namespace torch_tensorrt { namespace ptq { -bool get_batch_impl(void* bindings[], const char* names[], int nbBindings, torch::Tensor& data); +TORCHTRT_API bool get_batch_impl(void* bindings[], const char* names[], int nbBindings, torch::Tensor& data); } } // namespace torch_tensorrt #endif // DOXYGEN_SHOULD_SKIP_THIS @@ -513,7 +512,7 @@ }; template <typename Algorithm = nvinfer1::IInt8EntropyCalibrator2, typename DataLoader> -TORCHTRT_API inline Int8Calibrator<Algorithm, DataLoader> make_int8_calibrator( +inline Int8Calibrator<Algorithm, DataLoader> make_int8_calibrator( DataLoader dataloader, const std::string& cache_file_path, bool use_cache) { @@ -521,7 +520,7 @@ } template <typename Algorithm = nvinfer1::IInt8EntropyCalibrator2> -TORCHTRT_API inline Int8CacheCalibrator<Algorithm> make_int8_cache_calibrator(const std::string& cache_file_path) { +inline Int8CacheCalibrator<Algorithm> make_int8_cache_calibrator(const std::string& cache_file_path) { return Int8CacheCalibrator<Algorithm>(cache_file_path); } @@ -533,16 +532,16 @@
    - +
    @@ -574,12 +573,12 @@
    + + - - - + @@ -588,9 +587,9 @@ + - - + @@ -601,7 +600,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.html b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.html index 5b991abd75..6f868c9f99 100644 --- a/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.html +++ b/docs/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.html @@ -9,22 +9,22 @@ - + Program Listing for File torch_tensorrt.h — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • Program Listing for File torch_tensorrt.h
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +

    Program Listing for File torch_tensorrt.h

    ↰ Return to documentation for file (cpp/include/torch_tensorrt/torch_tensorrt.h)

    @@ -366,6 +364,9 @@ #include <set> #include <string> #include <vector> +#include "torch/custom_class.h" + +#include "torch_tensorrt/macros.h" // Just include the .h? #ifndef DOXYGEN_SHOULD_SKIP_THIS @@ -388,9 +389,8 @@ } #endif // DOXYGEN_SHOULD_SKIP_THIS -#include "torch_tensorrt/macros.h" namespace torch_tensorrt { -class TORCHTRT_API DataType { +class DataType { public: enum Value : int8_t { kFloat, @@ -403,7 +403,7 @@ DataType() = default; constexpr DataType(Value t) : value(t) {} - DataType(c10::ScalarType t); + TORCHTRT_API DataType(c10::ScalarType t); operator Value() const { return value; } @@ -422,7 +422,7 @@ } private: - friend std::ostream& operator<<(std::ostream& os, const DataType& dtype); + friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const DataType& dtype); Value value; }; @@ -475,7 +475,7 @@ kDLA_STANDALONE, }; -class TORCHTRT_API TensorFormat { +class TensorFormat { public: enum Value : int8_t { kContiguous, @@ -485,7 +485,7 @@ TensorFormat() = default; constexpr TensorFormat(Value t) : value(t) {} - TensorFormat(at::MemoryFormat t); + TORCHTRT_API TensorFormat(at::MemoryFormat t); operator Value() const { return value; } @@ -504,11 +504,11 @@ } private: - friend std::ostream& operator<<(std::ostream& os, const TensorFormat& format); + friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const TensorFormat& format); Value value; }; -struct TORCHTRT_API Input { +struct TORCHTRT_API Input : torch::CustomClassHolder { std::vector<int64_t> min_shape; std::vector<int64_t> opt_shape; std::vector<int64_t> max_shape; @@ -516,47 +516,53 @@ DataType dtype; TensorFormat format; - Input(std::vector<int64_t> shape, TensorFormat format = TensorFormat::kContiguous); + Input() {} + TORCHTRT_API Input(std::vector<int64_t> shape, TensorFormat format = TensorFormat::kContiguous); - Input(std::vector<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); + TORCHTRT_API Input(std::vector<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); - Input(c10::ArrayRef<int64_t> shape, TensorFormat format = TensorFormat::kContiguous); + TORCHTRT_API Input(c10::ArrayRef<int64_t> shape, TensorFormat format = TensorFormat::kContiguous); - Input(c10::ArrayRef<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); + TORCHTRT_API Input(c10::ArrayRef<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); - Input( + TORCHTRT_API Input( std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous); - Input( + TORCHTRT_API Input( std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); - Input( + TORCHTRT_API Input( c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous); - Input( + TORCHTRT_API Input( c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous); - Input(at::Tensor tensor); + TORCHTRT_API Input(at::Tensor tensor); private: - friend std::ostream& operator<<(std::ostream& os, const Input& input); + friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const Input& input); bool input_is_dynamic; }; +struct TORCHTRT_API GraphInputs { + torch::jit::IValue input_signature; // nested Input, full input spec + std::vector<Input> inputs; // flatten input spec +}; + TORCHTRT_API std::string get_build_info(); TORCHTRT_API void dump_build_info(); @@ -564,17 +570,17 @@ TORCHTRT_API void set_device(const int gpu_id); namespace torchscript { -struct TORCHTRT_API CompileSpec { - CompileSpec(std::vector<std::vector<int64_t>> fixed_sizes); +struct CompileSpec { + TORCHTRT_API CompileSpec(std::vector<std::vector<int64_t>> fixed_sizes); - CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes); + TORCHTRT_API CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes); - CompileSpec(std::vector<Input> inputs) : inputs(std::move(inputs)) {} + CompileSpec(std::vector<Input> inputs); + CompileSpec(torch::jit::IValue input_signature); // Defaults should reflect TensorRT defaults for BuilderConfig - std::vector<Input> inputs; - + GraphInputs graph_inputs; std::set<DataType> enabled_precisions = {DataType::kFloat}; bool disable_tf32 = false; @@ -591,11 +597,16 @@ EngineCapability capability = EngineCapability::kSTANDARD; - uint64_t num_min_timing_iters = 2; uint64_t num_avg_timing_iters = 1; uint64_t workspace_size = 0; + uint64_t dla_sram_size = 1048576; + + uint64_t dla_local_dram_size = 1073741824; + + uint64_t dla_global_dram_size = 536870912; + nvinfer1::IInt8Calibrator* ptq_calibrator = nullptr; bool require_full_compilation = false; @@ -625,16 +636,16 @@
    - +
    @@ -666,12 +677,12 @@
    + + - - - + @@ -680,9 +691,9 @@ + - - + @@ -693,7 +704,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_cpp_api/structtorch__tensorrt_1_1Device.html b/docs/_cpp_api/structtorch__tensorrt_1_1Device.html index 495abf6549..fe724e84d1 100644 --- a/docs/_cpp_api/structtorch__tensorrt_1_1Device.html +++ b/docs/_cpp_api/structtorch__tensorrt_1_1Device.html @@ -9,22 +9,22 @@ - + Struct Device — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Struct Device

      @@ -521,25 +519,25 @@

      Struct Documentation - +

    @@ -578,12 +576,12 @@

    Struct Documentation

    + + - - - + @@ -592,9 +590,9 @@

    Struct Documentation + - - + @@ -605,7 +603,7 @@

    Struct Documentation + diff --git a/docs/_cpp_api/structtorch__tensorrt_1_1GraphInputs.html b/docs/_cpp_api/structtorch__tensorrt_1_1GraphInputs.html new file mode 100644 index 0000000000..87c6a2f674 --- /dev/null +++ b/docs/_cpp_api/structtorch__tensorrt_1_1GraphInputs.html @@ -0,0 +1,674 @@ + + + + + + + + + + + + + Struct GraphInputs — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + + + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Struct GraphInputs

    + +
    +

    Struct Documentation

    +
    +
    +struct torch_tensorrt::GraphInputs
    +

    A struct to hold complex inputs.

    +

    This struct can either hold a complex inputs of shape or a flattened one,

    +
    +

    Public Members

    +
    +
    +torch::jit::IValue input_signature
    +
    + +
    +
    +std::vector<Input> inputs
    +
    + +
    +
    + +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/_cpp_api/structtorch__tensorrt_1_1Input.html b/docs/_cpp_api/structtorch__tensorrt_1_1Input.html index e03342da12..c37f2c20c1 100644 --- a/docs/_cpp_api/structtorch__tensorrt_1_1Input.html +++ b/docs/_cpp_api/structtorch__tensorrt_1_1Input.html @@ -9,22 +9,22 @@ - + Struct Input — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,13 +34,13 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,30 +342,44 @@
    - - + +
    - +
    - +

    Struct Input

    +
    +

    Inheritance Relationships

    +
    +

    Base Type

    +
      +

    • public CustomClassHolder

      • +
    +
    +

    Struct Documentation

    -struct torch_tensorrt::Input
    +struct torch_tensorrt::Input : public CustomClassHolder

    A struct to hold an input range (used by TensorRT Optimization profile)

    This struct can either hold a single vector representing an input shape, signifying a static input shape or a set of three input shapes representing the min, optiminal and max input shapes allowed for the engine.

    Public Functions

    +
    +
    +inline Input()
    +
    +
    -Input(std::vector<int64_t> shape, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(std::vector<int64_t> shape, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object for static input size from vector, optional arguments allow the user to configure expected input shape tensor format. dtype (Expected data type for the input) defaults to PyTorch / traditional TRT convection (FP32 for FP32 only, FP16 for FP32 and FP16, FP32 for Int8)

    Parameters
    @@ -381,7 +393,7 @@

    Struct Documentation
    -Input(std::vector<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(std::vector<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object for static input size from vector, optional arguments allow the user to configure expected input shape tensor format.

    Parameters
    @@ -396,7 +408,7 @@

    Struct Documentation
    -Input(c10::ArrayRef<int64_t> shape, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(c10::ArrayRef<int64_t> shape, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object for static input size from c10::ArrayRef (the type produced by tensor.sizes()), vector, optional arguments allow the user to configure expected input shape tensor format dtype (Expected data type for the input) defaults to PyTorch / traditional TRT convection (FP32 for FP32 only, FP16 for FP32 and FP16, FP32 for Int8)

    Parameters
    @@ -410,7 +422,7 @@

    Struct Documentation
    -Input(c10::ArrayRef<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(c10::ArrayRef<int64_t> shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object for static input size from c10::ArrayRef (the type produced by tensor.sizes()), vector, optional arguments allow the user to configure expected input shape tensor format.

    Parameters
    @@ -425,7 +437,7 @@

    Struct Documentation
    -Input(std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object dynamic input size from c10::ArrayRef (the type produced by tensor.sizes()) for min, opt, and max supported sizes. dtype (Expected data type for the input) defaults to PyTorch / traditional TRT convection (FP32 for FP32 only, FP16 for FP32 and FP16, FP32 for Int8)

    Parameters
    @@ -441,7 +453,7 @@

    Struct Documentation
    -Input(std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(std::vector<int64_t> min_shape, std::vector<int64_t> opt_shape, std::vector<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object for a dynamic input size from vectors for minimum shape, optimal shape, and max shape supported sizes optional arguments allow the user to configure expected input shape tensor format.

    Parameters
    @@ -458,7 +470,7 @@

    Struct Documentation
    -Input(c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object dynamic input size from c10::ArrayRef (the type produced by tensor.sizes()) for min, opt, and max supported sizes. dtype (Expected data type for the input) defaults to PyTorch / traditional TRT convection (FP32 for FP32 only, FP16 for FP32 and FP16, FP32 for Int8)

    Parameters
    @@ -474,7 +486,7 @@

    Struct Documentation
    -Input(c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)
    +TORCHTRT_API Input(c10::ArrayRef<int64_t> min_shape, c10::ArrayRef<int64_t> opt_shape, c10::ArrayRef<int64_t> max_shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous)

    Construct a new Input spec object dynamic input size from c10::ArrayRef (the type produced by tensor.sizes()) for min, opt, and max supported sizes.

    Parameters
    @@ -491,7 +503,7 @@

    Struct Documentation
    -Input(at::Tensor tensor)
    +TORCHTRT_API Input(at::Tensor tensor)

    Construct a new Input spec object using a torch tensor as an example The tensor’s shape, type and layout inform the spec’s values.

    Note: You cannot set dynamic shape through this method, you must use an alternative constructor

    @@ -548,25 +560,25 @@

    Struct Documentation - +

    @@ -590,6 +602,10 @@

    Struct Documentation

    + + - - - + @@ -615,9 +631,9 @@

    Struct Documentation + - - + @@ -628,7 +644,7 @@

    Struct Documentation + diff --git a/docs/_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.html b/docs/_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.html index d6b97c22f5..3bbaefa0d3 100644 --- a/docs/_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.html +++ b/docs/_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.html @@ -9,22 +9,22 @@ - + Struct CompileSpec — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    Struct CompileSpec

      @@ -366,7 +364,7 @@

      Struct DocumentationPublic Functions

      -CompileSpec(std::vector<std::vector<int64_t>> fixed_sizes)
      +TORCHTRT_API CompileSpec(std::vector<std::vector<int64_t>> fixed_sizes)

      Construct a new Compile Spec object Convienence constructor to set fixed input size from vectors describing size of input tensors. Each entry in the vector represents a input and should be provided in call order.

      This constructor should be use as a convience in the case that all inputs are static sized and you are okay with default input dtype and formats (FP32 for FP32 and INT8 weights, FP16 for FP16 weights, contiguous)

      @@ -378,8 +376,8 @@

      Struct Documentation
      -CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes)
      -

      Construct a new Extra Info object Convienence constructor to set fixed input size from c10::ArrayRef’s (the output of tensor.sizes()) describing size of input tensors. Each entry in the vector represents a input and should be provided in call order.

      +TORCHTRT_API CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes)
      +

      Construct a new Compile Spec object Convienence constructor to set fixed input size from c10::ArrayRef’s (the output of tensor.sizes()) describing size of input tensors. Each entry in the vector represents a input and should be provided in call order.

      This constructor should be use as a convience in the case that all inputs are static sized and you are okay with default input dtype and formats (FP32 for FP32 and INT8 weights, FP16 for FP16 weights, contiguous)

      Parameters
      @@ -390,8 +388,8 @@

      Struct Documentation
      -inline CompileSpec(std::vector<Input> inputs)
      -

      Construct a new Extra Info object from input ranges. Each entry in the vector represents a input and should be provided in call order.

      +CompileSpec(std::vector<Input> inputs)
      +

      Construct a new Compile Spec object from input ranges. Each entry in the vector represents a input and should be provided in call order.

      Use this constructor to define inputs with dynamic shape, specific input types or tensor formats

      Parameters
      @@ -400,14 +398,24 @@

      Struct Documentation

      +
      +
      +CompileSpec(torch::jit::IValue input_signature)
      +

      Construct a new Compile Spec object from IValue which represents the nesting of input tensors for a module.

      +
      +
      Parameters
      +

      input_signature –

      +
      +
      +
      +

    Public Members

    -
    -std::vector<Input> inputs
    -

    Specifications for inputs to the engine, can either be a single size or a range defined by min, opt and max sizes Users can also specify expected input type as well as tensor memory format.

    -

    Order in vector should match call order for the function

    +
    +GraphInputs graph_inputs
    +

    Specifications for inputs to the engine, can store a IValue which has stored complex Input or a flatened Input.

    @@ -459,12 +467,6 @@

    Struct Documentation

    Sets the restrictions for the engine (CUDA Safety)

    -
    -
    -uint64_t num_min_timing_iters = 2
    -

    Number of minimization timing iterations used to select kernels

    -
    -
    uint64_t num_avg_timing_iters = 1
    @@ -477,6 +479,24 @@

    Struct Documentation

    Maximum size of workspace given to TensorRT

    +
    +
    +uint64_t dla_sram_size = 1048576
    +

    Fast software managed RAM used by DLA to communicate within a layer.

    +
    + +
    +
    +uint64_t dla_local_dram_size = 1073741824
    +

    Host RAM used by DLA to share intermediate tensor data across operations

    +
    + +
    +
    +uint64_t dla_global_dram_size = 536870912
    +

    host RAM used by DLA to store weights and metadata for execution

    +
    +
    nvinfer1::IInt8Calibrator *ptq_calibrator = nullptr
    @@ -515,25 +535,25 @@

    Struct Documentation - +

    @@ -568,12 +588,12 @@

    Struct Documentation

    + + - - - + @@ -582,9 +602,9 @@

    Struct Documentation + - - + @@ -595,7 +615,7 @@

    Struct Documentation + diff --git a/docs/_cpp_api/torch_tensort_cpp.html b/docs/_cpp_api/torch_tensort_cpp.html index 52088c3328..d1bf077cde 100644 --- a/docs/_cpp_api/torch_tensort_cpp.html +++ b/docs/_cpp_api/torch_tensort_cpp.html @@ -9,22 +9,22 @@ - + Torch-TensorRT C++ API — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,13 +34,13 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Torch-TensorRT C++ API
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    Torch-TensorRT C++ API

    @@ -356,7 +354,7 @@

    Class Hierarchy

    • @@ -428,7 +426,19 @@

    Classes and Structs +

    +
    +
    + - - +
    - +

    @@ -740,11 +750,11 @@

    DefinesSphinx using a theme provided by Read the Docs.

    - + @@ -786,12 +796,12 @@

    Defines @@ -800,9 +810,9 @@

    Defines + - - + @@ -813,7 +823,7 @@

    Defines - + Full API — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +

    Full API

    @@ -444,16 +442,16 @@

    Files

    - +
    @@ -493,12 +491,12 @@

    Files

    + + - - - + @@ -507,9 +505,9 @@

    Files + - - + @@ -520,7 +518,7 @@

    Files jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_modules/index.html b/docs/_modules/index.html index a495476bb0..91b280d94c 100644 --- a/docs/_modules/index.html +++ b/docs/_modules/index.html @@ -6,24 +6,24 @@ - + - + Overview: module code — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,11 +33,11 @@ - + + - - + @@ -177,9 +177,9 @@ + - - +
    Table of Contents @@ -190,21 +190,21 @@
    @@ -285,7 +283,7 @@
    - + @@ -304,26 +302,26 @@
    - +
    @@ -335,19 +333,22 @@
    - - + +
    - +
    - +
    + - - +
    - +

    @@ -372,11 +373,11 @@

    All modules for which code is available

    - +
    Built with Sphinx using a theme provided by Read the Docs.
    - +
    @@ -386,19 +387,19 @@

    All modules for which code is available

    - +
    + + - - - + @@ -407,9 +408,9 @@

    All modules for which code is available

    + - - + @@ -420,7 +421,7 @@

    All modules for which code is available

    jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/_modules/torch_tensorrt/_Device.html b/docs/_modules/torch_tensorrt/_Device.html index c99734f7a3..3787cd84fa 100644 --- a/docs/_modules/torch_tensorrt/_Device.html +++ b/docs/_modules/torch_tensorrt/_Device.html @@ -6,24 +6,24 @@ - + - + torch_tensorrt._Device — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,11 +33,11 @@ - + + - - + @@ -177,9 +177,9 @@ + - - +
    Table of Contents @@ -190,21 +190,21 @@
    @@ -285,7 +283,7 @@
    - + @@ -304,28 +302,28 @@
    - +
    @@ -337,13 +335,13 @@
    - - + +
    - +
    - +

    Source code for torch_tensorrt._Device

     import torch
 
@@ -371,7 +369,7 @@ 
    Source code for torch_tensorrt._Device
         allow_gpu_fallback = False  #: (bool) Whether falling back to GPU if DLA cannot support an op should be allowed
 
 
    [docs]    def __init__(self, *args, **kwargs):
-        """ __init__ Method for torch_tensorrt.Device
+        """__init__ Method for torch_tensorrt.Device
 
         Device accepts one of a few construction patterns
 
@@ -393,7 +391,9 @@ 
    Source code for torch_tensorrt._Device
             """
         if len(args) == 1:
             if not isinstance(args[0], str):
-                raise TypeError("When specifying Device through positional argument, argument must be str")
+                raise TypeError(
+                    "When specifying Device through positional argument, argument must be str"
+                )
             else:
                 (self.device_type, id) = Device._parse_device_str(args[0])
                 if self.device_type == _enums.DeviceType.GPU:
@@ -401,8 +401,10 @@ 
    Source code for torch_tensorrt._Device
                     else:
                     self.dla_core = id
                     self.gpu_id = 0
-                    logging.log(logging.Level.Warning,
-                                "Setting GPU id to 0 for device because device 0 manages DLA on Xavier")
+                    logging.log(
+                        logging.Level.Warning,
+                        "Setting GPU id to 0 for device because device 0 manages DLA on Xavier",
+                    )
 
         elif len(args) == 0:
             if "gpu_id" in kwargs or "dla_core" in kwargs:
@@ -413,8 +415,10 @@ 
    Source code for torch_tensorrt._Device
                             self.gpu_id = kwargs["gpu_id"]
                     else:
                         self.gpu_id = 0
-                        logging.log(logging.Level.Warning,
-                                    "Setting GPU id to 0 for device because device 0 manages DLA on Xavier")
+                        logging.log(
+                            logging.Level.Warning,
+                            "Setting GPU id to 0 for device because device 0 manages DLA on Xavier",
+                        )
                 else:
                     self.gpu_id = kwargs["gpu_id"]
                     self.device_type = _enums.DeviceType.GPU
@@ -425,8 +429,10 @@ 
    Source code for torch_tensorrt._Device
     
         else:
             raise ValueError(
-                "Unexpected number of positional arguments for class Device \n    Found {} arguments, expected either zero or a single positional arguments"
-                .format(len(args)))
+                "Unexpected number of positional arguments for class Device \n    Found {} arguments, expected either zero or a single positional arguments".format(
+                    len(args)
+                )
+            )
 
         if "allow_gpu_fallback" in kwargs:
             if not isinstance(kwargs["allow_gpu_fallback"], bool):
@@ -434,8 +440,13 @@ 
    Source code for torch_tensorrt._Device
                 self.allow_gpu_fallback = kwargs["allow_gpu_fallback"]
    
 
     def __str__(self) -> str:
-        return "Device(type={}, gpu_id={}".format(self.device_type, self.gpu_id) \
-            + ")" if self.device_type == _enums.DeviceType.GPU else ", dla_core={}, allow_gpu_fallback={}".format(self.dla_core, self.allow_gpu_fallback)
+        return (
+            "Device(type={}, gpu_id={}".format(self.device_type, self.gpu_id) + ")"
+            if self.device_type == _enums.DeviceType.GPU
+            else ", dla_core={}, allow_gpu_fallback={}".format(
+                self.dla_core, self.allow_gpu_fallback
+            )
+        )
 
     def _to_internal(self) -> _C.Device:
         internal_dev = _C.Device()
@@ -447,8 +458,8 @@ 
    Source code for torch_tensorrt._Device
     
     @classmethod
     def _from_torch_device(cls, torch_dev: torch.device):
-        if torch_dev.type != 'cuda':
-            raise ValueError("Torch Device specs must have type \"cuda\"")
+        if torch_dev.type != "cuda":
+            raise ValueError('Torch Device specs must have type "cuda"')
         gpu_id = torch_dev.index
         return cls(gpu_id=gpu_id)
 
@@ -464,7 +475,7 @@ 
    Source code for torch_tensorrt._Device
         @staticmethod
     def _parse_device_str(s):
         s = s.lower()
-        spec = s.split(':')
+        spec = s.split(":")
         if spec[0] == "gpu" or spec[0] == "cuda":
             return (_enums.DeviceType.GPU, int(spec[1]))
         elif spec[0] == "dla":
@@ -472,16 +483,16 @@ 
    Source code for torch_tensorrt._Device
    - +
    + - - +
    - +

    @@ -489,11 +500,11 @@

    Source code for torch_tensorrt._Device
    - +
    Built with Sphinx using a theme provided by Read the Docs.
    - +
    @@ -503,19 +514,19 @@

    Source code for torch_tensorrt._Device

             
    
           
    
             
    
-
+
    + + - - - + @@ -524,9 +535,9 @@

    Source code for torch_tensorrt._Device

              
          
          
+     
 
-
-
+  
 
   
   
@@ -537,7 +548,7 @@ 
    Source code for torch_tensorrt._Device
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/_Input.html b/docs/_modules/torch_tensorrt/_Input.html
index 35ecbf05b9..e913dce4e6 100644
--- a/docs/_modules/torch_tensorrt/_Input.html
+++ b/docs/_modules/torch_tensorrt/_Input.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt._Input — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,13 +335,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt._Input
     from enum import Enum
 from typing import List, Dict, Any
@@ -377,12 +375,16 @@ 
    Source code for torch_tensorrt._Input
     
     shape_mode = None  #: (torch_tensorrt.Input._ShapeMode): Is input statically or dynamically shaped
     shape = None  #: (Tuple or Dict): Either a single Tuple or a dict of tuples defining the input shape. Static shaped inputs will have a single tuple. Dynamic inputs will have a dict of the form ``{ "min_shape": Tuple, "opt_shape": Tuple, "max_shape": Tuple }``
-    dtype = _enums.dtype.unknown  #: The expected data type of the input tensor (default: torch_tensorrt.dtype.float32)
+    dtype = (
+        _enums.dtype.unknown
+    )  #: The expected data type of the input tensor (default: torch_tensorrt.dtype.float32)
     _explicit_set_dtype = False
-    format = _enums.TensorFormat.contiguous  #: The expected format of the input tensor (default: torch_tensorrt.TensorFormat.NCHW)
+    format = (
+        _enums.TensorFormat.contiguous
+    )  #: The expected format of the input tensor (default: torch_tensorrt.TensorFormat.NCHW)
 
 
    [docs]    def __init__(self, *args, **kwargs):
-        """ __init__ Method for torch_tensorrt.Input
+        """__init__ Method for torch_tensorrt.Input
 
         Input accepts one of a few construction patterns
 
@@ -409,7 +411,8 @@ 
    Source code for torch_tensorrt._Input
                 if not Input._supported_input_size_type(args[0]):
                 raise TypeError(
                     "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                    + str(type(args[0])))
+                    + str(type(args[0]))
+                )
             if any(k in kwargs for k in ["min_shape", "opt_shape", "max_shape"]):
                 raise ValueError(
                     "Found that both shape (as a positional argument), and one or more of min_shape, opt_shape, max_shape were specified\nclass Input expects that only either shape or all three of min_shape, opt_shape, max_shape are defined"
@@ -418,11 +421,15 @@ 
    Source code for torch_tensorrt._Input
                 self.shape_mode = Input._ShapeMode.STATIC
 
         elif len(args) == 0:
-            if not ("shape" in kwargs) and not (all(k in kwargs for k in ["min_shape", "opt_shape", "max_shape"])):
+            if not ("shape" in kwargs) and not (
+                all(k in kwargs for k in ["min_shape", "opt_shape", "max_shape"])
+            ):
                 raise ValueError(
                     "Missing required arguments for class Input\nEither shape or all three of min_shape, opt_shape, max_shape must be defined"
                 )
-            elif ("shape" in kwargs) and all(k in kwargs for k in ["min_shape", "opt_shape", "max_shape"]):
+            elif ("shape" in kwargs) and all(
+                k in kwargs for k in ["min_shape", "opt_shape", "max_shape"]
+            ):
                 raise ValueError(
                     "Found that both shape, and one or more of min_shape, opt_shape, max_shape were specified\nclass Input expects that only either shape or all three of min_shape, opt_shape, max_shape are defined"
                 )
@@ -431,34 +438,43 @@ 
    Source code for torch_tensorrt._Input
                     if not Input._supported_input_size_type(kwargs["shape"]):
                     raise TypeError(
                         "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                        + str(type(kwargs["shape"])))
+                        + str(type(kwargs["shape"]))
+                    )
                 self.shape = tuple(kwargs["shape"])
                 self.shape_mode = Input._ShapeMode.STATIC
             else:
                 if not Input._supported_input_size_type(kwargs["min_shape"]):
                     raise TypeError(
                         "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                        + str(type(kwargs["min_shape"])) + " for min_shape")
+                        + str(type(kwargs["min_shape"]))
+                        + " for min_shape"
+                    )
                 if not Input._supported_input_size_type(kwargs["opt_shape"]):
                     raise TypeError(
                         "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                        + str(type(kwargs["opt_shape"])) + " for opt_shape")
+                        + str(type(kwargs["opt_shape"]))
+                        + " for opt_shape"
+                    )
                 if not Input._supported_input_size_type(kwargs["max_shape"]):
                     raise TypeError(
                         "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                        + str(type(kwargs["max_shape"])) + " for max_shape")
+                        + str(type(kwargs["max_shape"]))
+                        + " for max_shape"
+                    )
 
                 self.shape = {
                     "min_shape": tuple(kwargs["min_shape"]),
                     "opt_shape": tuple(kwargs["opt_shape"]),
-                    "max_shape": tuple(kwargs["max_shape"])
+                    "max_shape": tuple(kwargs["max_shape"]),
                 }
                 self.shape_mode = Input._ShapeMode.DYNAMIC
 
         else:
             raise ValueError(
-                "Unexpected number of positional arguments for class Input \n    Found {} arguments, expected either zero or a single positional arguments"
-                .format(len(args)))
+                "Unexpected number of positional arguments for class Input \n    Found {} arguments, expected either zero or a single positional arguments".format(
+                    len(args)
+                )
+            )
 
         if "dtype" in kwargs:
             self.dtype = Input._parse_dtype(kwargs["dtype"])
@@ -469,11 +485,17 @@ 
    Source code for torch_tensorrt._Input
     
     def __str__(self) -> str:
         if self.shape_mode == Input._ShapeMode.STATIC:
-            return "Input(shape={}, dtype={}, format={})".format(self.shape, str(self.dtype), str(self.format))
+            return "Input(shape={}, dtype={}, format={})".format(
+                self.shape, str(self.dtype), str(self.format)
+            )
         elif self.shape_mode == Input._ShapeMode.DYNAMIC:
             return "Input(min_shape={}, opt_shape={}, max_shape={}, dtype={}, format={})".format(
-                self.shape["min_shape"], self.shape["opt_shape"], self.shape["max_shape"], str(self.dtype),
-                str(self.format))
+                self.shape["min_shape"],
+                self.shape["opt_shape"],
+                self.shape["max_shape"],
+                str(self.dtype),
+                str(self.format),
+            )
         else:
             raise RuntimeError("Unknown input shape mode")
 
@@ -483,21 +505,27 @@ 
    Source code for torch_tensorrt._Input
                 if not Input._supported_input_size_type(self.shape["min_shape"]):
                 raise TypeError(
                     "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                    + str(type(self.shape["min_shape"])) + " for min_shape")
+                    + str(type(self.shape["min_shape"]))
+                    + " for min_shape"
+                )
             else:
                 internal_in.min = self.shape["min_shape"]
 
             if not Input._supported_input_size_type(self.shape["opt_shape"]):
                 raise TypeError(
                     "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                    + str(type(self.shape["opt_shape"])) + " for opt_shape")
+                    + str(type(self.shape["opt_shape"]))
+                    + " for opt_shape"
+                )
             else:
                 internal_in.opt = self.shape["opt_shape"]
 
             if not Input._supported_input_size_type(self.shape["max_shape"]):
                 raise TypeError(
                     "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                    + str(type(self.shape["max_shape"])) + " for max_shape")
+                    + str(type(self.shape["max_shape"]))
+                    + " for max_shape"
+                )
             else:
                 internal_in.max = self.shape["max_shape"]
             internal_in.input_is_dynamic = True
@@ -505,7 +533,9 @@ 
    Source code for torch_tensorrt._Input
                 if not Input._supported_input_size_type(self.shape):
                 raise TypeError(
                     "Input shape specifications for inputs are required to be a List, tuple or torch.Size, found type: "
-                    + str(type(self.shape)) + " for shape")
+                    + str(type(self.shape))
+                    + " for shape"
+                )
             else:
                 internal_in.opt = self.shape
             internal_in.input_is_dynamic = False
@@ -545,15 +575,17 @@ 
    Source code for torch_tensorrt._Input
                 else:
                 raise TypeError(
                     "Provided an unsupported data type as an input data type (support: bool, int32, half, float), got: "
-                    + str(dtype))
+                    + str(dtype)
+                )
 
         elif isinstance(dtype, _enums.dtype):
             return dtype
 
         else:
             raise TypeError(
-                "Input data type needs to be specified with a torch.dtype or a torch_tensorrt.dtype, got: " +
-                str(type(dtype)))
+                "Input data type needs to be specified with a torch.dtype or a torch_tensorrt.dtype, got: "
+                + str(type(dtype))
+            )
 
     @staticmethod
     def _parse_format(format: Any) -> _enums.TensorFormat:
@@ -564,40 +596,47 @@ 
    Source code for torch_tensorrt._Input
                     return _enums.TensorFormat.channels_last
             else:
                 raise ValueError(
-                    "Provided an unsupported tensor format (support: NHCW/contiguous_format, NHWC/channel_last)")
+                    "Provided an unsupported tensor format (support: NHCW/contiguous_format, NHWC/channel_last)"
+                )
 
         elif isinstance(format, _enums.TensorFormat):
             return format
 
         else:
             raise TypeError(
-                "Tensor format needs to be specified with either torch.memory_format or torch_tensorrt.TensorFormat")
+                "Tensor format needs to be specified with either torch.memory_format or torch_tensorrt.TensorFormat"
+            )
 
     @classmethod
     def _from_tensor(cls, t: torch.Tensor):
-        if not any([
+        if not any(
+            [
                 t.is_contiguous(memory_format=torch.contiguous_format),
-                t.is_contiguous(memory_format=torch.channels_last)
-        ]):
+                t.is_contiguous(memory_format=torch.channels_last),
+            ]
+        ):
             raise ValueError(
                 "Tensor does not have a supported contiguous memory format, supported formats are contiguous or channel_last"
             )
-        frmt = torch.contiguous_format if t.is_contiguous(
-            memory_format=torch.contiguous_format) else torch.channels_last
+        frmt = (
+            torch.contiguous_format
+            if t.is_contiguous(memory_format=torch.contiguous_format)
+            else torch.channels_last
+        )
         return cls(shape=t.shape, dtype=t.dtype, format=frmt)
    
 
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -605,11 +644,11 @@ 
    Source code for torch_tensorrt._Input
     
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -619,19 +658,19 @@ 
    Source code for torch_tensorrt._Input
             
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -640,9 +679,9 @@ 
    Source code for torch_tensorrt._Input
              
          
          
+     
 
-
-
+  
 
   
   
@@ -653,7 +692,7 @@ 
    Source code for torch_tensorrt._Input
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/_compile.html b/docs/_modules/torch_tensorrt/_compile.html
index 1706166fc5..b2d0a4c3bf 100644
--- a/docs/_modules/torch_tensorrt/_compile.html
+++ b/docs/_modules/torch_tensorrt/_compile.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt._compile — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,40 +335,47 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt._compile
     from typing import List, Dict, Any
 from torch_tensorrt import _enums
 import torch_tensorrt.ts
 from torch_tensorrt import logging
 import torch
-from torch import fx
+import torch.fx
 from enum import Enum
-from torch_tensorrt import fx
+
+import torch_tensorrt.fx
+import torch_tensorrt.fx.lower
+from torch_tensorrt.fx.utils import LowerPrecision
+
 
 class _IRType(Enum):
-    """Enum to set the minimum required logging level to print a message to stdout
-    """
+    """Enum to set the minimum required logging level to print a message to stdout"""
+
     ts = 0
     fx = 1
 
 
 class _ModuleType(Enum):
-    """Enum to set the minimum required logging level to print a message to stdout
-    """
+    """Enum to set the minimum required logging level to print a message to stdout"""
+
     nn = 0
     ts = 1
     fx = 2
 
 
 def _parse_module_type(module: Any) -> _ModuleType:
-    if any(isinstance(module, t) for t in [torch.jit.ScriptModule, torch.jit.ScriptFunction]):
+    if any(
+        isinstance(module, t)
+        for t in [torch.jit.ScriptModule, torch.jit.ScriptFunction]
+    ):
         return _ModuleType.ts
     elif isinstance(module, torch.fx.GraphModule):
         return _ModuleType.fx
@@ -395,19 +400,29 @@ 
    Source code for torch_tensorrt._compile
             if ir == "default":
             # Options are listed in order of preference
             if module_is_tsable:
-                logging.log(logging.Level.Info, "ir was set to default, using TorchScript as ir")
+                logging.log(
+                    logging.Level.Info, "ir was set to default, using TorchScript as ir"
+                )
                 return _IRType.ts
             elif module_is_fxable:
-                raise ValueError("Was given a torch.fx.GraphModule, fx is not currently supported by Torch-TensorRT")
-                #logging.log(logging.Level.Info, "ir was set to default, using TorchScript as fx")
-                #return _IRType.fx
+                raise ValueError(
+                    "Was given a torch.fx.GraphModule, fx is not currently supported by Torch-TensorRT"
+                )
+                # logging.log(logging.Level.Info, "ir was set to default, using TorchScript as fx")
+                # return _IRType.fx
             else:
                 raise ValueError("Module was provided with in an unsupported format")
         else:
             raise ValueError("Unknown ir was requested")
 
 
-
    [docs]def compile(module: Any, ir="default", inputs=[], enabled_precisions=set([_enums.dtype.float]), **kwargs):
+
    [docs]def compile(
+    module: Any,
+    ir="default",
+    inputs=[],
+    enabled_precisions=set([_enums.dtype.float]),
+    **kwargs,
+):
     """Compile a PyTorch module for NVIDIA GPUs using TensorRT
 
     Takes a existing PyTorch module and a set of settings to configure the compiler
@@ -450,93 +465,46 @@ 
    Source code for torch_tensorrt._compile
             if module_type == _ModuleType.nn:
             logging.log(
                 logging.Level.Info,
-                "Module was provided as a torch.nn.Module, trying to script the module with torch.jit.script. In the event of a failure please preconvert your module to TorchScript"
+                "Module was provided as a torch.nn.Module, trying to script the module with torch.jit.script. In the event of a failure please preconvert your module to TorchScript",
             )
             ts_mod = torch.jit.script(module)
-        return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
+        return torch_tensorrt.ts.compile(
+            ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs
+        )
     elif target_ir == _IRType.fx:
-        from torch_tensorrt.fx.tracer.acc_tracer import acc_tracer
-        from torch_tensorrt.fx import InputTensorSpec
-        from torch_tensorrt.fx import TRTInterpreter
-        from torch_tensorrt.fx.passes.lower_basic_pass import transform_setitem
-        from torch_tensorrt.fx.tools.trt_splitter import TRTSplitter
-        from torch_tensorrt.fx.tools.trt_splitter import TRTSplitterSetting
-        from torch_tensorrt.fx.trt_module import TRTModule
-        from torch_tensorrt.fx.utils import LowerPrecision
-        acc_model = acc_tracer.trace(module, inputs)
-
-        splitter_setting = TRTSplitterSetting()
-        splitter_setting.use_implicit_batch_dim = False
-        splitter = TRTSplitter(acc_model, inputs, settings=splitter_setting)
-        splitter.node_support_preview()
-        split_mod = splitter()
-        num_piece = 0
-        for name, _ in split_mod.named_children():
-            print(f"graph is split into {name}")
-            num_piece += 1
-
-        # if the graph module is split into pieces larger than 8, we consider its perf
-        # is not good and fall back to non-TRT
-        if num_piece > 8:
-            print(
-                f"The graph module is split into {num_piece} which is large than the \
-                threshold=8. Fall back to non-TRT module."
-            )
-            return None
-
-        if torch.float16 in enabled_precisions or torch.half in enabled_precisions:
-            precision = LowerPrecision.FP16
+        if (
+            torch.float16 in enabled_precisions
+            or torch_tensorrt.dtype.half in enabled_precisions
+        ):
+            lower_precision = LowerPrecision.FP16
+        elif (
+            torch.float32 in enabled_precisions
+            or torch_tensorrt.dtype.float in enabled_precisions
+        ):
+            lower_precision = LowerPrecision.FP32
         else:
-            precision = LowerPrecision.FP32
-
-        def get_submod_inputs(mod, submod, inputs):
-            acc_inputs = None
-
-            def get_input(self, inputs):
-                nonlocal acc_inputs
-                acc_inputs = inputs
-
-            handle = submod.register_forward_pre_hook(get_input)
-            mod(*inputs)
-            handle.remove()
-            return acc_inputs
-
-        for name, _ in split_mod.named_children():
-            if "_run_on_acc" in name:
-                submod = getattr(split_mod, name)
-                # Get submodule inputs for fx2trt
-                acc_inputs = get_submod_inputs(split_mod, submod, inputs)
-
-                # fx2trt replacement
-                interp = TRTInterpreter(
-                    submod,
-                    InputTensorSpec.from_tensors(acc_inputs),
-                    explicit_batch_dimension=True,
-                )
-                r = interp.run(
-                    max_workspace_size=20 << 30,
-                    lower_precision=precision,
-                    # profiling_verbosity=trt.ProfilingVerbosity.DETAILED, #For profile
-                )
-                # For profile
-                # from torch_tensorrt.fx.tools.trt_profiler_sorted import profile_trt_module
-                # profile_trt_module("", trt_mod, acc_inputs)
-                trt_mod = TRTModule(*r)
-
-                setattr(split_mod, name, trt_mod)
-            else:
-                submod = getattr(split_mod, name)
-        return split_mod
+            raise ValueError(f"Precision {enabled_precisions} not supported on FX")
+
+        return torch_tensorrt.fx.lower.compile(
+            module,
+            inputs,
+            lower_precision=lower_precision,
+            max_batch_size=inputs[0].size(0),
+            explicit_batch_dimension=True,
+            dynamic_batch=False,
+        )
     else:
         raise RuntimeError("Module is an unknown format or the ir requested is unknown")
    
 
 
-
    [docs]def convert_method_to_trt_engine(module: Any,
-                                 method_name: str,
-                                 ir="default",
-                                 inputs=[],
-                                 enabled_precisions=set([_enums.dtype.float]),
-                                 **kwargs):
+
    [docs]def convert_method_to_trt_engine(
+    module: Any,
+    method_name: str,
+    ir="default",
+    inputs=[],
+    enabled_precisions=set([_enums.dtype.float]),
+    **kwargs,
+):
     """Convert a TorchScript module method to a serialized TensorRT engine
 
     Converts a specified method of a module to a serialized TensorRT engine given a dictionary of conversion settings
@@ -574,14 +542,16 @@ 
    Source code for torch_tensorrt._compile
             if module_type == _ModuleType.nn:
             logging.log(
                 logging.Level.Info,
-                "Module was provided as a torch.nn.Module, trying to script the module with torch.jit.script. In the event of a failure please preconvert your module to TorchScript"
+                "Module was provided as a torch.nn.Module, trying to script the module with torch.jit.script. In the event of a failure please preconvert your module to TorchScript",
             )
             ts_mod = torch.jit.script(module)
-        return torch_tensorrt.ts.convert_method_to_trt_engine(ts_mod,
-                                                              method_name,
-                                                              inputs=inputs,
-                                                              enabled_precisions=enabled_precisions,
-                                                              **kwargs)
+        return torch_tensorrt.ts.convert_method_to_trt_engine(
+            ts_mod,
+            method_name,
+            inputs=inputs,
+            enabled_precisions=enabled_precisions,
+            **kwargs,
+        )
     elif target_ir == _IRType.fx:
         raise RuntimeError("fx is currently not supported")
     else:
@@ -589,16 +559,16 @@ 
    Source code for torch_tensorrt._compile
     
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -606,11 +576,11 @@ 
    Source code for torch_tensorrt._compile
     
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -620,19 +590,19 @@ 
    Source code for torch_tensorrt._compile
             
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -641,9 +611,9 @@ 
    Source code for torch_tensorrt._compile
              
          
          
+     
 
-
-
+  
 
   
   
@@ -654,7 +624,7 @@ 
    Source code for torch_tensorrt._compile
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/_util.html b/docs/_modules/torch_tensorrt/_util.html
index a5e7a79e5b..f0c0df9c15 100644
--- a/docs/_modules/torch_tensorrt/_util.html
+++ b/docs/_modules/torch_tensorrt/_util.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt._util — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,21 +335,22 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt._util
     from torch_tensorrt import __version__
 from torch_tensorrt import _C
 
+import torch
+
 
 
    [docs]def dump_build_info():
-    """Prints build information about the torch_tensorrt distribution to stdout
-    """
+    """Prints build information about the torch_tensorrt distribution to stdout"""
     print(get_build_info())
    
 
 
@@ -362,7 +361,15 @@ 
    Source code for torch_tensorrt._util
             str: String containing the build information for torch_tensorrt distribution
     """
     build_info = _C.get_build_info()
-    build_info = "Torch-TensorRT Version: " + str(__version__) + '\n' + build_info
+    build_info = (
+        "Torch-TensorRT Version: "
+        + str(__version__)
+        + "\n"
+        + "Using PyTorch Version: "
+        + str(torch.__version__)
+        + "\n"
+        + build_info
+    )
     return build_info
    
 
 
@@ -371,16 +378,16 @@ 
    Source code for torch_tensorrt._util
     
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -388,11 +395,11 @@ 
    Source code for torch_tensorrt._util
     
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -402,19 +409,19 @@ 
    Source code for torch_tensorrt._util
             
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -423,9 +430,9 @@ 
    Source code for torch_tensorrt._util
              
          
          
+     
 
-
-
+  
 
   
   
@@ -436,7 +443,7 @@ 
    Source code for torch_tensorrt._util
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/fx/fx2trt.html b/docs/_modules/torch_tensorrt/fx/fx2trt.html
new file mode 100644
index 0000000000..27b503f746
--- /dev/null
+++ b/docs/_modules/torch_tensorrt/fx/fx2trt.html
@@ -0,0 +1,999 @@
+
+
+
+
+
+  
+
+  
+  
+  
+  
+  torch_tensorrt.fx.fx2trt — Torch-TensorRT master documentation
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+  
+  
+  
+    
+    
+  
+  
+  
+  
+
+  
+  
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+
+
+
+
+   
+
+    
+
+    
    
+      Table of Contents
+      
+    
    
+
+    
+
+    
    
+      
    
+        
    
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    
+
+  
+
+  
+
    
+        
    
+
+        
    
+          Shortcuts
+        
    
+      
    
+
+      
    
+        
    
+
+        
+          
+          
    
+          
+            
    
+             
    
+              
+  
    Source code for torch_tensorrt.fx.fx2trt
    +import logging
+import warnings
+from datetime import datetime
+from typing import Any, Callable, Dict, List, NamedTuple, Optional, Sequence
+
+import numpy
+
+# @manual=//deeplearning/trt/python:py_tensorrt
+import tensorrt as trt
+import torch
+import torch.fx
+from torch.fx.node import _get_qualified_name
+from torch.fx.passes.shape_prop import TensorMetadata
+
+from .converter_registry import CONVERTERS
+from .input_tensor_spec import InputTensorSpec
+from .observer import Observer
+from .utils import get_dynamic_dims, LowerPrecision, torch_dtype_to_trt
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+TRT_INTERPRETER_CALL_PRE_OBSERVER: Observer[
+    Callable[[torch.fx.GraphModule], None]
+] = Observer("TRT_INTERPRETER_CALL_PRE_OBSERVER")
+
+
+
    [docs]class TRTInterpreterResult(NamedTuple):
+    engine: Any
+    input_names: Sequence[str]
+    output_names: Sequence[str]
+    serialized_cache: bytearray
    
+
+
+
    [docs]class TRTInterpreter(torch.fx.Interpreter):
+    def __init__(
+        self,
+        module: torch.fx.GraphModule,
+        input_specs: List[InputTensorSpec],
+        explicit_batch_dimension: bool = False,
+        explicit_precision: bool = False,
+        logger_level=None,
+    ):
+        super().__init__(module)
+
+        self.logger = trt.Logger(logger_level or trt.Logger.WARNING)
+        self.builder = trt.Builder(self.logger)
+
+        flag = 0
+        if explicit_batch_dimension:
+            EXPLICIT_BATCH = 1 << (int)(
+                trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH
+            )
+            flag |= EXPLICIT_BATCH
+
+        if explicit_precision:
+            EXPLICIT_PRECISION = 1 << (int)(
+                trt.NetworkDefinitionCreationFlag.EXPLICIT_PRECISION
+            )
+            flag |= EXPLICIT_PRECISION
+        self.network = self.builder.create_network(flag)
+
+        missing_ops = self.validate_conversion()
+        if missing_ops:
+            warnings.warn(
+                "Interpretation will fail due to missing operations \n"
+                + "\n".join(f"{i}" for i in missing_ops)
+            )
+
+        self.optimization_profiles: Optional[List] = None
+        self.input_specs = input_specs
+        self.input_specs_iter = 0
+        self.validate_input_specs()
+        self._cur_node_name: Optional[str] = None
+        self._input_names: List[str] = []
+        self._output_names: List[str] = []
+        self._itensor_to_tensor_meta: Dict[
+            trt.tensorrt.ITensor, TensorMetadata
+        ] = dict()
+
+    def validate_input_specs(self):
+        for shape, _, _, shape_ranges, has_batch_dim in self.input_specs:
+            if not self.network.has_implicit_batch_dimension:
+                assert (
+                    has_batch_dim
+                ), "It's required to specify batch dimension when it's explicit in TensorRT network."
+
+            dynamic_dims = get_dynamic_dims(shape)
+            if len(dynamic_dims):
+                assert not self.network.has_implicit_batch_dimension, (
+                    "Can't have dynamic dim when "
+                    f"batch dim is implicit, got {shape}."
+                )
+                assert len(
+                    shape_ranges
+                ), "shape_ranges must be provided when shape has dynamic dim."
+
+                if self.optimization_profiles:
+                    assert len(shape_ranges) == len(self.optimization_profiles), (
+                        "Number of optimization "
+                        f"profiles {len(self.optimization_profiles)} doesn't match with the number of shape_range"
+                        f" {len(shape_ranges)} provided."
+                    )
+                else:
+                    self.optimization_profiles = [
+                        self.builder.create_optimization_profile()
+                        for _ in range(len(shape_ranges))
+                    ]
+
+                for shape_range in shape_ranges:
+                    assert (
+                        len(shape_range) == 3
+                    ), f"Expect three elements in shape_range, got {len(shape_range)}"
+                    assert all(len(s) == len(shape) for s in shape_range), (
+                        "Expect elements in shape_range"
+                        f" {shape_range} have the same number of dimension as the provided shape {len(shape)}"
+                    )
+
+                    for i in range(len(shape)):
+                        if i in dynamic_dims:
+                            assert all(
+                                shape_range[j][i] <= shape_range[j + 1][i]
+                                for j in range(2)
+                            ), (
+                                "Expect dynamic dim"
+                                f" {i} to have incremental value for shapes in shape_range {shape_range}."
+                            )
+                        else:
+                            assert all(s[i] == shape[i] for s in shape_range), (
+                                f"Expect non dynamic dim {i} to be the same"
+                                f" for all shapes in shape_range {shape_range}."
+                            )
+            else:
+                assert (
+                    len(shape_ranges) == 0
+                ), "shape_ranges are provided for input that doesn't have dynamic dim."
+
+    def validate_conversion(self):
+        missing_converter = set()
+
+        for node in self.module.graph.nodes:
+            if node.op == "call_function" and not CONVERTERS.get(node.target):
+                missing_converter.add(f"{node.op} {_get_qualified_name(node.target)}")
+            elif node.op == "call_method" and not CONVERTERS.get(node.target):
+                missing_converter.add(f"{node.op} torch.Tensor.{node.target}")
+            elif node.op == "call_module":
+                submod = self.fetch_attr(node.target)
+                submod_type = getattr(submod, "_base_class_origin", type(submod))
+                if not CONVERTERS.get(submod_type):
+                    missing_converter.add(f"{node.op} {torch.typename(submod_type)}")
+
+        return missing_converter
+
+    def run(
+        self,
+        max_batch_size=64,
+        max_workspace_size=1 << 25,
+        lower_precision=LowerPrecision.FP16,
+        sparse_weights=False,
+        force_fp32_output=False,
+        strict_type_constraints=False,
+        algorithm_selector=None,
+        timing_cache=None,
+        profiling_verbosity=None,
+        tactic_sources=None,
+    ) -> TRTInterpreterResult:
+        """
+        Build TensorRT engine with some configs.
+        Args:
+            max_batch_size: set accordingly for maximum batch size you will use.
+            max_workspace_size: set to the maximum size we can afford for temporary buffer
+            lower_precision: the precision model layers are running on (TensorRT will choose the best perforamnce precision).
+            sparse_weights: allow the builder to examine weights and use optimized functions when weights have suitable sparsity
+            force_fp32_output: force output to be fp32
+            strict_type_constraints: Usually we should set it to False unless we want to control the precision of certain layer for numeric reasons.
+            algorithm_selector: set up algorithm selection for certain layer
+            timing_cache: enable timing cache for TensorRT
+            profiling_verbosity: TensorRT logging level
+        Return:
+            TRTInterpreterResult
+        """
+        TRT_INTERPRETER_CALL_PRE_OBSERVER.observe(self.module)
+
+        # For float outputs, we set their dtype to fp16 only if lower_precision == LowerPrecision.FP16 and
+        # force_fp32_output=False.
+        self.output_fp16 = (
+            not force_fp32_output and lower_precision == LowerPrecision.FP16
+        )
+
+        if (
+            lower_precision == LowerPrecision.INT8
+            and not self.builder.platform_has_fast_int8
+        ):
+            raise RuntimeError("Current platform doesn't support fast native int8!")
+
+        if (
+            lower_precision == LowerPrecision.FP16
+            and not self.builder.platform_has_fast_fp16
+        ):
+            warnings.warn("Current platform doesn't support fast native fp16!")
+
+        self.input_specs_iter = 0
+        run_module_start_time = datetime.now()
+        super().run()
+        _LOGGER.info(
+            f"Run Module elapsed time: {datetime.now() - run_module_start_time}"
+        )
+        build_engine_start_time = datetime.now()
+
+        self.builder.max_batch_size = max_batch_size
+        builder_config = self.builder.create_builder_config()
+        builder_config.max_workspace_size = max_workspace_size
+
+        cache = None
+        if timing_cache:
+            cache_file = numpy.array(timing_cache)
+            cache = builder_config.create_timing_cache(cache_file.tobytes())
+        else:
+            cache = builder_config.create_timing_cache(b"")
+        builder_config.set_timing_cache(cache, False)
+
+        if trt.__version__ >= "8.2":
+            builder_config.profiling_verbosity = (
+                profiling_verbosity
+                if profiling_verbosity
+                else trt.ProfilingVerbosity.LAYER_NAMES_ONLY
+            )
+        if lower_precision == LowerPrecision.FP16:
+            builder_config.set_flag(trt.BuilderFlag.FP16)
+
+        if lower_precision == LowerPrecision.INT8:
+            builder_config.set_flag(trt.BuilderFlag.INT8)
+
+        if sparse_weights:
+            builder_config.set_flag(trt.BuilderFlag.SPARSE_WEIGHTS)
+
+        if strict_type_constraints:
+            builder_config.set_flag(trt.BuilderFlag.STRICT_TYPES)
+
+        if self.optimization_profiles:
+            for optimization_profile in self.optimization_profiles:
+                builder_config.add_optimization_profile(optimization_profile)
+
+        if algorithm_selector:
+            builder_config.set_flag(trt.BuilderFlag.DISABLE_TIMING_CACHE)
+            builder_config.algorithm_selector = algorithm_selector
+
+        if tactic_sources is not None:
+            builder_config.set_tactic_sources(tactic_sources=tactic_sources)
+
+        engine = self.builder.build_engine(self.network, builder_config)
+        assert engine
+
+        serialized_cache = (
+            bytearray(cache.serialize())
+            if builder_config.get_timing_cache()
+            else bytearray()
+        )
+        _LOGGER.info(
+            f"Build TRT engine elapsed time: {datetime.now() - build_engine_start_time}"
+        )
+
+        return TRTInterpreterResult(
+            engine, self._input_names, self._output_names, serialized_cache
+        )
+
+    def run_node(self, n):
+        self._cur_node_name = str(n)
+        # add "_itensor_to_tensor_meta"
+        kwargs = dict(n.kwargs)
+        kwargs["_itensor_to_tensor_meta"] = self._itensor_to_tensor_meta
+        n.kwargs = kwargs
+
+        # run the node
+        trt_node = super().run_node(n)
+
+        # remove "_itensor_to_tensor_meta"
+        kwargs = dict(n.kwargs)
+        del kwargs["_itensor_to_tensor_meta"]
+        n.kwargs = kwargs
+
+        if isinstance(trt_node, trt.tensorrt.ITensor):
+            self._itensor_to_tensor_meta[trt_node] = n.meta.get("tensor_meta")
+
+        return trt_node
+
+    def placeholder(self, target, args, kwargs):
+        self._input_names.append(target)
+        shape, dtype, _, shape_ranges, has_batch_dim = self.input_specs[
+            self.input_specs_iter
+        ]
+        self.input_specs_iter += 1
+
+        if self.network.has_implicit_batch_dimension:
+            if has_batch_dim:
+                shape = shape[1:]
+        else:
+            for i, shape_range in enumerate(shape_ranges):
+                assert self.optimization_profiles
+                self.optimization_profiles[i].set_shape(target, *shape_range)
+
+        return self.network.add_input(
+            name=target, shape=tuple(shape), dtype=torch_dtype_to_trt(dtype)
+        )
+
+    def call_module(self, target, args, kwargs):
+        assert isinstance(target, str)
+        submod = self.fetch_attr(target)
+        submod_type = getattr(submod, "_base_class_origin", type(submod))
+        converter = CONVERTERS.get(submod_type)
+
+        if not converter:
+            raise RuntimeError(
+                f"Conversion of module of type {submod_type} not currently supported!"
+            )
+
+        assert self._cur_node_name is not None
+        return converter(self.network, submod, args, kwargs, self._cur_node_name)
+
+    def call_function(self, target, args, kwargs):
+        converter = CONVERTERS.get(target)
+
+        if not converter:
+            raise RuntimeError(
+                f"Conversion of function {torch.typename(target)} not currently supported!"
+            )
+
+        assert self._cur_node_name is not None
+        return converter(self.network, target, args, kwargs, self._cur_node_name)
+
+    def call_method(self, target, args, kwargs):
+        assert isinstance(target, str)
+        converter = CONVERTERS.get(target)
+
+        if not converter:
+            raise RuntimeError(
+                f"Conversion of method {target} not currently supported!"
+            )
+
+        assert self._cur_node_name is not None
+        return converter(self.network, target, args, kwargs, self._cur_node_name)
+
+    def output(self, target, args, kwargs):
+        assert len(args) == 1
+        outputs = args[0] if isinstance(args[0], tuple) else (args[0],)
+
+        if not all(isinstance(output, trt.tensorrt.ITensor) for output in outputs):
+            raise RuntimeError("TensorRT requires all outputs to be Tensor!")
+
+        for i, output in enumerate(outputs):
+            if any(
+                op_name in output.name.split("_")
+                for op_name in (
+                    "eq",
+                    "gt",
+                    "lt",
+                    "or",
+                    "xor",
+                    "and",
+                    "not",
+                    "ne",
+                    "isinf",
+                    "any",
+                )
+            ):
+                output_bool = True
+            else:
+                output_bool = False
+            name = f"output{i}"
+            output.name = name
+            self.network.mark_output(output)
+            if output_bool:
+                output.dtype = trt.bool
+            elif self.output_fp16 and output.dtype == trt.float32:
+                output.dtype = trt.float16
+            self._output_names.append(name)
    
+
    
+
+             
    
+             
+            
    
+            
    
+  
+
+  
+
+    
    
+
+  
+
+  
    
+    
    
+        © Copyright 2021, NVIDIA Corporation.
+
+    
    
+  
    
+    
+      
    
+        Built with Sphinx using a theme provided by Read the Docs.
+      
    
+     
+
+
    
+
+          
    
+        
    
+
+        
    
+          
    
+            
    
+              
+            
    
+          
    
+        
    
+      
    
+    
    
+
+  
+
+
+  
+
+     
+       
+         
+         
+         
+         
+         
+         
+         
+     
+
+  
+
+  
+  
+  
+  
+
+   
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
    Docs
    
+          
    Access comprehensive developer documentation for PyTorch
    
+          View Docs
+        
    
+
+        
    
+          
    Tutorials
    
+          
    Get in-depth tutorials for beginners and advanced developers
    
+          View Tutorials
+        
    
+
+        
    
+          
    Resources
    
+          
    Find development resources and get your questions answered
    
+          View Resources
+        
    
+      
    
+    
    
+  
    
+
+  
+
+  
    
+  
    
+    
    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.
    
+    
+  
    
+
    
+
+  
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
+          
+        
    
+      
    
+    
    
+
+    
    
+      
    
+        
+      
    
+    
    
+  
    
+
+  
+
+  
+
+  
+
+
\ No newline at end of file
diff --git a/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html b/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html
new file mode 100644
index 0000000000..3a1aabfb63
--- /dev/null
+++ b/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html
@@ -0,0 +1,809 @@
+
+
+
+
+
+  
+
+  
+  
+  
+  
+  torch_tensorrt.fx.input_tensor_spec — Torch-TensorRT master documentation
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+  
+  
+  
+    
+    
+  
+  
+  
+  
+
+  
+  
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+
+
+
+
+   
+
+    
+
+    
    
+      Table of Contents
+      
+    
    
+
+    
+
+    
    
+      
    
+        
    
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    
+
+  
+
+  
+
    
+        
    
+
+        
    
+          Shortcuts
+        
    
+      
    
+
+      
    
+        
    
+
+        
+          
+          
    
+          
+            
    
+             
    
+              
+  
    Source code for torch_tensorrt.fx.input_tensor_spec
    +from typing import Iterable, List, NamedTuple, Optional, Sequence, Tuple
+
+import torch
+
+from .types import Shape, ShapeRange
+from .utils import get_dynamic_dims
+
+
+def generate_input_specs(inputs, lower_setting, additional_inputs=None):
+    # dynamic_batch is TRT only flag.
+    if (
+        not lower_setting.explicit_batch_dimension
+        or lower_setting.dynamic_batch is False
+    ):
+        return InputTensorSpec.from_tensors(inputs)
+
+    # If we don't have additional inputs, we assume the first dimension
+    # is the dynamic batch dimension. Otherwise, we use the additional
+    # inputs to determine the batch dimension.
+    if additional_inputs is None:
+        return InputTensorSpec.from_tensors_with_dynamic_batch_size(
+            inputs,
+            (
+                0,
+                lower_setting.max_batch_size,
+                lower_setting.max_batch_size,
+            ),
+            lower_setting.opt_profile_replica,
+        )
+    else:
+        batch_dims = []
+
+        for i, j in zip(inputs, additional_inputs):
+            found_batch_dim = False
+
+            for idx, values in enumerate(zip(i.shape, j.shape)):
+                if values[0] != values[1]:
+                    assert (
+                        found_batch_dim is False
+                    ), f"We've already found a batch dim, {i.shape}, {j.shape}."
+                    batch_dims.append(idx)
+                    found_batch_dim = True
+
+            if not found_batch_dim:
+                raise RuntimeError(
+                    f"Failed to find batch dimension because shapes are the same, {i.shape}"
+                )
+
+        return InputTensorSpec.from_tensors_with_dynamic_batch_size(
+            inputs,
+            (
+                0,
+                lower_setting.max_batch_size,
+                lower_setting.max_batch_size,
+            ),
+            lower_setting.opt_profile_replica,
+            batch_dims,
+        )
+
+
+
    [docs]class InputTensorSpec(NamedTuple):
+    """
+    This class contains the information of a input tensor.
+
+    shape: shape of the tensor.
+
+    dtype: dtyep of the tensor.
+
+    device: device of the tensor. This is only used to generate inputs to the given model
+        in order to run shape prop. For TensorRT engine, inputs have to be on cuda device.
+
+    shape_ranges: If dynamic shape is needed (shape has dimensions of -1), then this field
+        has to be provided (default is empty list). Every shape_range is a tuple of three
+        tuples ((min_input_shape), (optimized_input_shape), (max_input_shape)). Each shape_range
+        is used to populate a TensorRT optimization profile.
+        e.g. If the input shape varies from (1, 224) to (100, 224) and we want to optimize
+        for (25, 224) because it's the most common input shape, then we set shape_ranges to
+        ((1, 224), (25, 225), (100, 224)).
+
+    has_batch_dim: Whether the shape includes batch dimension. Batch dimension has to be provided
+        if the engine want to run with dynamic shape.
+    """
+
+    shape: Shape
+    dtype: torch.dtype
+    device: torch.device = torch.device("cpu")
+    shape_ranges: List[ShapeRange] = []
+    has_batch_dim: bool = True
+
+    @classmethod
+    def from_tensor(cls, tensor: torch.Tensor) -> "InputTensorSpec":
+        """
+        Produce an InputTenosrSpec named tuple which contains the
+        information of the given PyTorch tensor.
+
+        Args:
+            tensor (torch.Tensor): A PyTorch tensor.
+
+        Returns:
+            An InputTensorSpec named tuple.
+        """
+        return cls(tensor.shape, tensor.dtype, tensor.device)
+
+    @classmethod
+    def from_tensors(cls, tensors: Sequence[torch.Tensor]) -> List["InputTensorSpec"]:
+        """
+        Produce a list of InputTenosrSpec named tuples which contain
+        the information of all the given PyTorch tensors.
+
+        Args:
+            tensors (Iterable[torch.Tensor]): A list of PyTorch tensors.
+
+        Returns:
+            A list of InputTensorSpec named tuples.
+        """
+        assert isinstance(tensors, (list, tuple))
+        return [cls.from_tensor(t) for t in tensors]
+
+    @classmethod
+    def from_tensors_with_dynamic_batch_size(
+        cls,
+        tensors: Sequence[torch.Tensor],
+        batch_size_range: Tuple[int, int, int],
+        opt_profile_replica: int = 1,
+        batch_dims: Optional[List[int]] = None,
+    ) -> List["InputTensorSpec"]:
+        """
+        Produce a list of InputTenosrSpec named tuples which would contain
+        the information of all the given PyTorch tensors. The produced input
+        tensor specs will treat all tensors' first dimension as batch dimension
+        and mark them as dynmaic.
+
+        Args:
+            tensors (Sequence[torch.Tensor]): A list of PyTorch tensors.
+            batch_size_range (Tuple[int, int, int]): The first integer indicates
+                the smallest batch size allowed. The second integer indiceates
+                the batch size that we'll optimize for. The third integer indicates
+                the largest batch size allowed.
+            opt_profile_replica (int): If dynamic shape is enabled, each execution
+                context requires a different optimization profile. This arg determines
+                how many optimization profile replicas we want to produce.
+            batch_dims (Optional[List[int]]): The batch dim might not be the leading dim
+                and allow user to specify the batch dims using this arg. Default we treat
+                dim 0 as the batch dim.
+
+        Returns:
+            A list of InputTensorSpec named tuples with dynamic ranges.
+        """
+        if batch_dims is None:
+            batch_dims = [0] * len(tensors)
+
+        input_specs = []
+        batch_size = tensors[0].size(batch_dims[0])
+
+        for i, tensor in enumerate(tensors):
+            batch_dim = batch_dims[i]
+            assert batch_size == tensor.size(
+                batch_dim
+            ), f"The {i}th tensor (shape: {tensor.shape}) doesn't have the correct batch size: {batch_size}."
+            shape = list(tensor.shape)
+            shape[batch_dim] = -1
+            shape_ranges: List[ShapeRange] = [tuple(tuple(shape[0:batch_dim] + [bs] + shape[batch_dim + 1 :]) for bs in batch_size_range)] * opt_profile_replica  # type: ignore[list-item]
+            input_specs.append(
+                cls(tuple(shape), tensor.dtype, tensor.device, shape_ranges)
+            )
+
+        return input_specs
+
+    def to_random_tensor(self):
+        shape = tuple(self.shape)
+        if len(get_dynamic_dims(shape)):
+            shape = tuple(self.shape_ranges[0][1])
+        elif not self.has_batch_dim:
+            shape = (1,) + tuple(shape)
+
+        return torch.randn(shape).to(dtype=self.dtype, device=self.device)
+
+    @staticmethod
+    def create_inputs_from_specs(input_specs: Iterable["InputTensorSpec"]):
+        inputs = []
+
+        for spec in input_specs:
+            inputs.append(spec.to_random_tensor())
+
+        return inputs
    
+
    
+
+             
    
+             
+            
    
+            
    
+  
+
+  
+
+    
    
+
+  
+
+  
    
+    
    
+        © Copyright 2021, NVIDIA Corporation.
+
+    
    
+  
    
+    
+      
    
+        Built with Sphinx using a theme provided by Read the Docs.
+      
    
+     
+
+
    
+
+          
    
+        
    
+
+        
    
+          
    
+            
    
+              
+            
    
+          
    
+        
    
+      
    
+    
    
+
+  
+
+
+  
+
+     
+       
+         
+         
+         
+         
+         
+         
+         
+     
+
+  
+
+  
+  
+  
+  
+
+   
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
    Docs
    
+          
    Access comprehensive developer documentation for PyTorch
    
+          View Docs
+        
    
+
+        
    
+          
    Tutorials
    
+          
    Get in-depth tutorials for beginners and advanced developers
    
+          View Tutorials
+        
    
+
+        
    
+          
    Resources
    
+          
    Find development resources and get your questions answered
    
+          View Resources
+        
    
+      
    
+    
    
+  
    
+
+  
+
+  
    
+  
    
+    
    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.
    
+    
+  
    
+
    
+
+  
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
+          
+        
    
+      
    
+    
    
+
+    
    
+      
    
+        
+      
    
+    
    
+  
    
+
+  
+
+  
+
+  
+
+
\ No newline at end of file
diff --git a/docs/_modules/torch_tensorrt/fx/trt_module.html b/docs/_modules/torch_tensorrt/fx/trt_module.html
new file mode 100644
index 0000000000..0e8d1d35b3
--- /dev/null
+++ b/docs/_modules/torch_tensorrt/fx/trt_module.html
@@ -0,0 +1,863 @@
+
+
+
+
+
+  
+
+  
+  
+  
+  
+  torch_tensorrt.fx.trt_module — Torch-TensorRT master documentation
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+  
+  
+  
+    
+    
+  
+  
+  
+  
+
+  
+  
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+
+
+
+
+   
+
+    
+
+    
    
+      Table of Contents
+      
+    
    
+
+    
+
+    
    
+      
    
+        
    
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    
+
+  
+
+  
+
    
+        
    
+
+        
    
+          Shortcuts
+        
    
+      
    
+
+      
    
+        
    
+
+        
+          
+          
    
+          
+            
    
+             
    
+              
+  
    Source code for torch_tensorrt.fx.trt_module
    +from typing import Any, List, Sequence
+
+# @manual=//deeplearning/trt/python:py_tensorrt
+import tensorrt as trt
+import torch
+
+from .utils import torch_dtype_from_trt
+
+
+
    [docs]class TRTModule(torch.nn.Module):
+    def __init__(
+        self, engine=None, input_names=None, output_names=None, cuda_graph_batch_size=-1
+    ):
+        super(TRTModule, self).__init__()
+        self._register_state_dict_hook(TRTModule._on_state_dict)
+        self.engine = engine
+        self.input_names = input_names
+        self.output_names = output_names
+        self.cuda_graph_batch_size = cuda_graph_batch_size
+        self.initialized = False
+
+        if engine:
+            self._initialize()
+
+    def _initialize(self):
+        self.initialized = True
+        self.context = self.engine.create_execution_context()
+
+        # Indices of inputs/outputs in the trt engine bindings, in the order
+        # as they are in the original PyTorch model.
+        self.input_binding_indices_in_order: Sequence[int] = [
+            self.engine.get_binding_index(name) for name in self.input_names
+        ]
+        self.output_binding_indices_in_order: Sequence[int] = [
+            self.engine.get_binding_index(name) for name in self.output_names
+        ]
+        primary_input_outputs = set()
+        primary_input_outputs.update(self.input_binding_indices_in_order)
+        primary_input_outputs.update(self.output_binding_indices_in_order)
+        self.hidden_output_binding_indices_in_order: Sequence[int] = []
+        self.hidden_output_names: Sequence[str] = []
+        for i in range(
+            self.engine.num_bindings // self.engine.num_optimization_profiles
+        ):
+            if i not in primary_input_outputs:
+                self.hidden_output_binding_indices_in_order.append(i)
+                self.hidden_output_names.append(self.engine.get_binding_name(i))
+
+        assert (self.engine.num_bindings // self.engine.num_optimization_profiles) == (
+            len(self.input_names)
+            + len(self.output_names)
+            + len(self.hidden_output_names)
+        )
+
+        self.input_dtypes: Sequence[torch.dtype] = [
+            torch_dtype_from_trt(self.engine.get_binding_dtype(idx))
+            for idx in self.input_binding_indices_in_order
+        ]
+        self.input_shapes: Sequence[Sequence[int]] = [
+            tuple(self.engine.get_binding_shape(idx))
+            for idx in self.input_binding_indices_in_order
+        ]
+        self.output_dtypes: Sequence[torch.dtype] = [
+            torch_dtype_from_trt(self.engine.get_binding_dtype(idx))
+            for idx in self.output_binding_indices_in_order
+        ]
+        self.output_shapes = [
+            tuple(self.engine.get_binding_shape(idx))
+            if self.engine.has_implicit_batch_dimension
+            else tuple()
+            for idx in self.output_binding_indices_in_order
+        ]
+        self.hidden_output_dtypes: Sequence[torch.dtype] = [
+            torch_dtype_from_trt(self.engine.get_binding_dtype(idx))
+            for idx in self.hidden_output_binding_indices_in_order
+        ]
+        self.hidden_output_shapes = [
+            tuple(self.engine.get_binding_shape(idx))
+            if self.engine.has_implicit_batch_dimension
+            else tuple()
+            for idx in self.hidden_output_binding_indices_in_order
+        ]
+
+    def _check_initialized(self):
+        if not self.initialized:
+            raise RuntimeError("TRTModule is not initialized.")
+
+    def _on_state_dict(self, state_dict, prefix, local_metadata):
+        self._check_initialized()
+        state_dict[prefix + "engine"] = bytearray(self.engine.serialize())
+        state_dict[prefix + "input_names"] = self.input_names
+        state_dict[prefix + "output_names"] = self.output_names
+        state_dict[prefix + "cuda_graph_batch_size"] = self.cuda_graph_batch_size
+
+    def _load_from_state_dict(
+        self,
+        state_dict,
+        prefix,
+        local_metadata,
+        strict,
+        missing_keys,
+        unexpected_keys,
+        error_msgs,
+    ):
+        engine_bytes = state_dict[prefix + "engine"]
+
+        logger = trt.Logger()
+        runtime = trt.Runtime(logger)
+        self.engine = runtime.deserialize_cuda_engine(engine_bytes)
+
+        self.input_names = state_dict[prefix + "input_names"]
+        self.output_names = state_dict[prefix + "output_names"]
+        self._initialize()
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["engine"] = bytearray(self.engine.serialize())
+        state.pop("context", None)
+        return state
+
+    def __setstate__(self, state):
+        logger = trt.Logger()
+        runtime = trt.Runtime(logger)
+        state["engine"] = runtime.deserialize_cuda_engine(state["engine"])
+        self.__dict__.update(state)
+        if self.engine:
+            self.context = self.engine.create_execution_context()
+
+    def forward(self, *inputs):
+        with torch.autograd.profiler.record_function("TRTModule:Forward"):
+            self._check_initialized()
+
+            with torch.autograd.profiler.record_function("TRTModule:ProcessInputs"):
+                assert len(inputs) == len(
+                    self.input_names
+                ), f"Wrong number of inputs, expect {len(self.input_names)} get {len(inputs)}."
+
+                # This is only used when the trt engine is using implicit batch dim.
+                batch_size = inputs[0].shape[0]
+                contiguous_inputs: List[torch.Tensor] = [i.contiguous() for i in inputs]
+                bindings: List[Any] = [None] * (
+                    len(self.input_names)
+                    + len(self.output_names)
+                    + len(self.hidden_output_names)
+                )
+
+                for i, input_name in enumerate(self.input_names):
+                    assert inputs[
+                        i
+                    ].is_cuda, f"{i}th input({input_name}) is not on cuda device."
+                    assert (
+                        inputs[i].dtype == self.input_dtypes[i]
+                    ), f"Dtype mismatch for {i}th input({input_name}). Expect {self.input_dtypes[i]}, got {inputs[i].dtype}."
+
+                    idx = self.input_binding_indices_in_order[i]
+                    bindings[idx] = contiguous_inputs[i].data_ptr()
+
+                    if not self.engine.has_implicit_batch_dimension:
+                        self.context.set_binding_shape(
+                            idx, tuple(contiguous_inputs[i].shape)
+                        )
+                    else:
+                        assert inputs[i].size()[1:] == self.input_shapes[i], (
+                            f"Shape mismatch for {i}th input({input_name}). "
+                            f"Expect {self.input_shapes[i]}, got {inputs[i].size()[1:]}."
+                        )
+
+            with torch.autograd.profiler.record_function("TRTModule:ProcessOutputs"):
+                # create output tensors
+                outputs: List[torch.Tensor] = []
+
+                for i, idx in enumerate(self.output_binding_indices_in_order):
+                    if self.engine.has_implicit_batch_dimension:
+                        shape = (batch_size,) + self.output_shapes[i]
+                    else:
+                        shape = tuple(self.context.get_binding_shape(idx))
+
+                    output = torch.empty(  # type: ignore[call-overload]
+                        size=shape,
+                        dtype=self.output_dtypes[i],
+                        device=torch.cuda.current_device(),
+                    )
+                    outputs.append(output)
+                    bindings[idx] = output.data_ptr()
+
+                for i, idx in enumerate(self.hidden_output_binding_indices_in_order):
+                    if self.engine.has_implicit_batch_dimension:
+                        shape = (batch_size,) + self.hidden_output_shapes[i]
+                    else:
+                        shape = tuple(self.context.get_binding_shape(idx))
+
+                    output = torch.empty(  # type: ignore[call-overload]
+                        size=shape,
+                        dtype=self.hidden_output_dtypes[i],
+                        device=torch.cuda.current_device(),
+                    )
+                    bindings[idx] = output.data_ptr()
+
+            with torch.autograd.profiler.record_function("TRTModule:TensorRTRuntime"):
+                if self.engine.has_implicit_batch_dimension:
+                    self.context.execute_async(
+                        batch_size, bindings, torch.cuda.current_stream().cuda_stream
+                    )
+                else:
+                    self.context.execute_async_v2(
+                        bindings, torch.cuda.current_stream().cuda_stream
+                    )
+
+            if len(outputs) == 1:
+                return outputs[0]
+
+            return tuple(outputs)
+
+    def enable_profiling(self, profiler: "trt.IProfiler" = None):
+        """
+        Enable TensorRT profiling. After calling this function, TensorRT will report
+        time spent on each layer in stdout for each forward run.
+        """
+        self._check_initialized()
+
+        if not self.context.profiler:
+            self.context.profiler = trt.Profiler() if profiler is None else profiler
+
+    def disable_profiling(self):
+        """
+        Disable TensorRT profiling.
+        """
+        self._check_initialized()
+
+        torch.cuda.synchronize()
+        del self.context
+        self.context = self.engine.create_execution_context()
+
+    def get_layer_info(self) -> str:
+        """
+        Get layer info of the engine. Only support for TRT > 8.2.
+        """
+        inspector = self.engine.create_engine_inspector()
+        return inspector.get_engine_information(trt.LayerInformationFormat.JSON)
    
+
    
+
+             
    
+             
+            
    
+            
    
+  
+
+  
+
+    
    
+
+  
+
+  
    
+    
    
+        © Copyright 2021, NVIDIA Corporation.
+
+    
    
+  
    
+    
+      
    
+        Built with Sphinx using a theme provided by Read the Docs.
+      
    
+     
+
+
    
+
+          
    
+        
    
+
+        
    
+          
    
+            
    
+              
+            
    
+          
    
+        
    
+      
    
+    
    
+
+  
+
+
+  
+
+     
+       
+         
+         
+         
+         
+         
+         
+         
+     
+
+  
+
+  
+  
+  
+  
+
+   
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
    Docs
    
+          
    Access comprehensive developer documentation for PyTorch
    
+          View Docs
+        
    
+
+        
    
+          
    Tutorials
    
+          
    Get in-depth tutorials for beginners and advanced developers
    
+          View Tutorials
+        
    
+
+        
    
+          
    Resources
    
+          
    Find development resources and get your questions answered
    
+          View Resources
+        
    
+      
    
+    
    
+  
    
+
+  
+
+  
    
+  
    
+    
    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.
    
+    
+  
    
+
    
+
+  
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
+          
+        
    
+      
    
+    
    
+
+    
    
+      
    
+        
+      
    
+    
    
+  
    
+
+  
+
+  
+
+  
+
+
\ No newline at end of file
diff --git a/docs/_modules/torch_tensorrt/logging.html b/docs/_modules/torch_tensorrt/logging.html
index 337d0c9ca2..f4c1f72a6f 100644
--- a/docs/_modules/torch_tensorrt/logging.html
+++ b/docs/_modules/torch_tensorrt/logging.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt.logging — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,24 +335,30 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt.logging
     from enum import Enum
-from torch_tensorrt._C import _get_logging_prefix, _set_logging_prefix, \
-                              _get_reportable_log_level, _set_reportable_log_level, \
-                              _get_is_colored_output_on,  _set_is_colored_output_on, \
-                               _log, LogLevel
+from torch_tensorrt._C import (
+    _get_logging_prefix,
+    _set_logging_prefix,
+    _get_reportable_log_level,
+    _set_reportable_log_level,
+    _get_is_colored_output_on,
+    _set_is_colored_output_on,
+    _log,
+    LogLevel,
+)
 
 
 
    [docs]class Level(Enum):
-    """Enum to set the minimum required logging level to print a message to stdout
-    """
+    """Enum to set the minimum required logging level to print a message to stdout"""
+
     InternalError = LogLevel.INTERNAL_ERROR
     Error = LogLevel.ERROR
     Warning = LogLevel.WARNING
@@ -556,16 +560,16 @@ 
    Source code for torch_tensorrt.logging
     
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -573,11 +577,11 @@ 
    Source code for torch_tensorrt.logging
     
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -587,19 +591,19 @@ 
    Source code for torch_tensorrt.logging
             
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -608,9 +612,9 @@ 
    Source code for torch_tensorrt.logging
              
          
          
+     
 
-
-
+  
 
   
   
@@ -621,7 +625,7 @@ 
    Source code for torch_tensorrt.logging
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/ptq.html b/docs/_modules/torch_tensorrt/ptq.html
index 0f0c5235c0..29740e7958 100644
--- a/docs/_modules/torch_tensorrt/ptq.html
+++ b/docs/_modules/torch_tensorrt/ptq.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt.ptq — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,13 +335,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt.ptq
     from typing import List, Dict, Any
 import torch
@@ -377,12 +375,13 @@ 
    Source code for torch_tensorrt.ptq
     
     batch = self.dataset_iterator.next()
     self.current_batch_idx += self.batch_size
-    # Treat the first element as input and others as targets.
+    inputs_gpu = []
     if isinstance(batch, list):
-        batch = batch[0].to(self.device)
+        for example in batch:
+            inputs_gpu.append(example.to(self.device).data_ptr())
     else:
-        batch = batch.to(self.device)
-    return [batch.data_ptr()]
    
+        inputs_gpu.append(batch.to(self.device).data_ptr())
+    return inputs_gpu
    
 
 
 
    [docs]def read_calibration_cache(self):
@@ -425,46 +424,64 @@ 
    Source code for torch_tensorrt.ptq
             device = kwargs.get("device", torch.device("cuda:0"))
 
         if not isinstance(dataloader, torch.utils.data.DataLoader):
-            log(Level.Error,
-                "Dataloader : {} is not a valid instance of torch.utils.data.DataLoader".format(dataloader))
+            log(
+                Level.Error,
+                "Dataloader : {} is not a valid instance of torch.utils.data.DataLoader".format(
+                    dataloader
+                ),
+            )
 
         if not cache_file:
             if use_cache:
-                log(Level.Debug, "Using existing cache_file {} for calibration".format(cache_file))
+                log(
+                    Level.Debug,
+                    "Using existing cache_file {} for calibration".format(cache_file),
+                )
             else:
                 log(Level.Debug, "Overwriting existing calibration cache file.")
         else:
             if use_cache:
-                log(Level.Error, "Input cache file is None but use_cache is set to True in INT8 mode.")
+                log(
+                    Level.Error,
+                    "Input cache file is None but use_cache is set to True in INT8 mode.",
+                )
 
         # Define attributes and member functions for the calibrator class
         attribute_mapping = {
-            'data_loader': dataloader,
-            'current_batch_idx': 0,
-            'batch_size': dataloader.batch_size,
-            'dataset_iterator': iter(dataloader),
-            'cache_file': cache_file,
-            'device': device,
-            'use_cache': use_cache,
-            'get_batch_size': get_batch_size,
-            'get_batch': get_cache_mode_batch if use_cache else get_batch,
-            'read_calibration_cache': read_calibration_cache,
-            'write_calibration_cache': write_calibration_cache
+            "data_loader": dataloader,
+            "current_batch_idx": 0,
+            "batch_size": dataloader.batch_size,
+            "dataset_iterator": iter(dataloader),
+            "cache_file": cache_file,
+            "device": device,
+            "use_cache": use_cache,
+            "get_batch_size": get_batch_size,
+            "get_batch": get_cache_mode_batch if use_cache else get_batch,
+            "read_calibration_cache": read_calibration_cache,
+            "write_calibration_cache": write_calibration_cache,
         }
 
         # Using type metaclass to construct calibrator class based on algorithm type
         if algo_type == CalibrationAlgo.ENTROPY_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8EntropyCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8EntropyCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.ENTROPY_CALIBRATION_2:
-            return type('DataLoaderCalibrator', (_C.IInt8MinMaxCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8MinMaxCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.LEGACY_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8LegacyCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8LegacyCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.MINMAX_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8MinMaxCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8MinMaxCalibrator,), attribute_mapping
+            )()
         else:
             log(
                 Level.Error,
-                "Invalid calibration algorithm type. Please select among ENTROPY_CALIBRATION, ENTROPY_CALIBRATION, LEGACY_CALIBRATION or MINMAX_CALIBRATION"
+                "Invalid calibration algorithm type. Please select among ENTROPY_CALIBRATION, ENTROPY_CALIBRATION, LEGACY_CALIBRATION or MINMAX_CALIBRATION",
             )
    
 
 
@@ -484,46 +501,57 @@ 
    Source code for torch_tensorrt.ptq
             algo_type = kwargs.get("algo_type", CalibrationAlgo.ENTROPY_CALIBRATION_2)
 
         if os.path.isfile(cache_file):
-            log(Level.Debug, "Using existing cache_file {} for calibration".format(cache_file))
+            log(
+                Level.Debug,
+                "Using existing cache_file {} for calibration".format(cache_file),
+            )
         else:
             log(Level.Error, "Invalid calibration cache file.")
 
         # Define attributes and member functions for the calibrator class
         attribute_mapping = {
-            'use_cache': True,
-            'cache_file': cache_file,
-            'get_batch_size': get_batch_size,
-            'get_batch': get_cache_mode_batch,
-            'read_calibration_cache': read_calibration_cache,
-            'write_calibration_cache': write_calibration_cache
+            "use_cache": True,
+            "cache_file": cache_file,
+            "get_batch_size": get_batch_size,
+            "get_batch": get_cache_mode_batch,
+            "read_calibration_cache": read_calibration_cache,
+            "write_calibration_cache": write_calibration_cache,
         }
         # Using type metaclass to construct calibrator class based on algorithm type
         if algo_type == CalibrationAlgo.ENTROPY_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8EntropyCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8EntropyCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.ENTROPY_CALIBRATION_2:
-            return type('DataLoaderCalibrator', (_C.IInt8MinMaxCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8MinMaxCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.LEGACY_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8LegacyCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8LegacyCalibrator,), attribute_mapping
+            )()
         elif algo_type == CalibrationAlgo.MINMAX_CALIBRATION:
-            return type('DataLoaderCalibrator', (_C.IInt8MinMaxCalibrator,), attribute_mapping)()
+            return type(
+                "DataLoaderCalibrator", (_C.IInt8MinMaxCalibrator,), attribute_mapping
+            )()
         else:
             log(
                 Level.Error,
-                "Invalid calibration algorithm type. Please select among ENTROPY_CALIBRATION, ENTROPY_CALIBRATION, LEGACY_CALIBRATION or MINMAX_CALIBRATION"
+                "Invalid calibration algorithm type. Please select among ENTROPY_CALIBRATION, ENTROPY_CALIBRATION, LEGACY_CALIBRATION or MINMAX_CALIBRATION",
             )

    - +
    + - - +
    - +

    @@ -531,11 +559,11 @@

    Source code for torch_tensorrt.ptq
    - +
    Built with Sphinx using a theme provided by Read the Docs.
    - +
    @@ -545,19 +573,19 @@

    Source code for torch_tensorrt.ptq

             
    
           
    
             
    
-
+
    + + - - - + @@ -566,9 +594,9 @@

    Source code for torch_tensorrt.ptq

              
          
          
+     
 
-
-
+  
 
   
   
@@ -579,7 +607,7 @@ 
    Source code for torch_tensorrt.ptq
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/ts/_compile_spec.html b/docs/_modules/torch_tensorrt/ts/_compile_spec.html
index f0249dcff5..adce31756f 100644
--- a/docs/_modules/torch_tensorrt/ts/_compile_spec.html
+++ b/docs/_modules/torch_tensorrt/ts/_compile_spec.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt.ts._compile_spec — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,13 +335,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt.ts._compile_spec
     from typing import List, Dict, Any, Set
 import torch
@@ -352,8 +350,22 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
 from torch_tensorrt import _enums
 from torch_tensorrt._Input import Input
 from torch_tensorrt._Device import Device
-
+from torch_tensorrt.logging import Level, log
+from typing import Tuple, List, Dict
 import warnings
+from copy import deepcopy
+
+
+def _internal_input_to_torch_class_input(i: _C.Input) -> torch.classes.tensorrt._Input:
+    clone = torch.classes.tensorrt._Input()
+    clone._set_min(i.min)
+    clone._set_opt(i.opt)
+    clone._set_max(i.max)
+    clone._set_dtype(i.dtype)
+    clone._set_format(i.format)
+    clone._set_input_is_dynamic(i.input_is_dynamic)
+    clone._set_explicit_set_dtype(i._explicit_set_dtype)
+    return clone
 
 
 def _supported_input_size_type(input_size: Any) -> bool:
@@ -366,27 +378,37 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
     else:
         raise TypeError(
             "Input sizes for inputs are required to be a List, tuple or torch.Size or a Dict of three sizes (min, opt, max), found type: "
-            + str(type(input_size)))
+            + str(type(input_size))
+        )
 
 
 def _parse_input_ranges(input_sizes: List) -> List:
 
-    if any(not isinstance(i, dict) and not _supported_input_size_type(i) for i in input_sizes):
-        raise KeyError("An input size must either be a static size or a range of three sizes (min, opt, max) as Dict")
+    if any(
+        not isinstance(i, dict) and not _supported_input_size_type(i)
+        for i in input_sizes
+    ):
+        raise KeyError(
+            "An input size must either be a static size or a range of three sizes (min, opt, max) as Dict"
+        )
 
     parsed_input_sizes = []
     for i in input_sizes:
         if isinstance(i, dict):
             if all(k in i for k in ["min", "opt", "min"]):
                 parsed_input_sizes.append(
-                    Input(min_shape=i["min"], opt_shape=i["opt"], max_shape=i["max"])._to_internal())
+                    Input(
+                        min_shape=i["min"], opt_shape=i["opt"], max_shape=i["max"]
+                    )._to_internal()
+                )
 
             elif "opt" in i:
                 parsed_input_sizes.append(Input(shape=i["opt"])._to_internal())
 
             else:
                 raise KeyError(
-                    "An input size must either be a static size or a range of three sizes (min, opt, max) as Dict")
+                    "An input size must either be a static size or a range of three sizes (min, opt, max) as Dict"
+                )
 
         elif isinstance(i, list):
             parsed_input_sizes.append(Input(shape=i)._to_internal())
@@ -409,15 +431,19 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         elif precision == torch.float:
             return _enums.dtype.float
         else:
-            raise TypeError("Provided an unsupported dtype as operating precision (support: int8, half, float), got: " +
-                            str(precision))
+            raise TypeError(
+                "Provided an unsupported dtype as operating precision (support: int8, half, float), got: "
+                + str(precision)
+            )
 
     elif isinstance(precision, _enums.dtype):
         return precision
 
     else:
-        raise TypeError("Op precision type needs to be specified with a torch.dtype or a torch_tensorrt.dtype, got: " +
-                        str(type(precision)))
+        raise TypeError(
+            "Op precision type needs to be specified with a torch.dtype or a torch_tensorrt.dtype, got: "
+            + str(type(precision))
+        )
 
 
 def _parse_enabled_precisions(precisions: Any) -> Set:
@@ -432,10 +458,14 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
 
 def _parse_device_type(device: Any) -> _enums.DeviceType:
     if isinstance(device, torch.device):
-        if device.type == 'cuda':
+        if device.type == "cuda":
             return _enums.DeviceType.gpu
         else:
-            ValueError("Got a device type other than GPU or DLA (type: " + str(device.type) + ")")
+            ValueError(
+                "Got a device type other than GPU or DLA (type: "
+                + str(device.type)
+                + ")"
+            )
     elif isinstance(device, _enums.DeviceType):
         return device
     elif isinstance(device, str):
@@ -444,11 +474,14 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         elif device == "dla" or device == "DLA":
             return _enums.DeviceType.dla
         else:
-            ValueError("Got a device type other than GPU or DLA (type: " + str(device) + ")")
+            ValueError(
+                "Got a device type other than GPU or DLA (type: " + str(device) + ")"
+            )
     else:
         raise TypeError(
-            "Device specification must be of type torch.device, string or torch_tensorrt.DeviceType, but got: " +
-            str(type(device)))
+            "Device specification must be of type torch.device, string or torch_tensorrt.DeviceType, but got: "
+            + str(type(device))
+        )
 
 
 def _parse_device(device_info: Any) -> _C.Device:
@@ -479,7 +512,8 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         return (Device._from_torch_device(device_info))._to_internal()
     else:
         raise ValueError(
-            "Unsupported data for device specification. Expected either a dict, torch_tensorrt.Device or torch.Device")
+            "Unsupported data for device specification. Expected either a dict, torch_tensorrt.Device or torch.Device"
+        )
 
 
 def _parse_torch_fallback(fallback_info: Dict[str, Any]) -> _ts_C.TorchFallback:
@@ -504,25 +538,113 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
     return info
 
 
-def _parse_compile_spec(compile_spec: Dict[str, Any]) -> _ts_C.CompileSpec:
-    info = _ts_C.CompileSpec()
-    if "inputs" not in compile_spec:
+def _parse_input_signature(input_signature: Any):
+    if isinstance(input_signature, tuple):
+        input_list = []
+        for item in input_signature:
+            input = _parse_input_signature(item)
+            input_list.append(input)
+        return tuple(input_list)
+    elif isinstance(input_signature, list):
+        input_list = []
+        for item in input_signature:
+            input = _parse_input_signature(item)
+            input_list.append(input)
+        return input_list
+    elif isinstance(input_signature, Input) or isinstance(
+        input_signature, torch.Tensor
+    ):
+        i = (
+            Input._from_tensor(input_signature)
+            if isinstance(input_signature, torch.Tensor)
+            else input_signature
+        )
+        clone = _internal_input_to_torch_class_input(i._to_internal())
+        return clone
+    else:
         raise KeyError(
-            "Module input definitions are requried to compile module. Provide a list of torch_tensorrt.Input keyed to \"inputs\" in the compile spec"
+            "Input signature contains an unsupported type {}".format(
+                type(input_signature)
+            )
         )
 
-    if "inputs" in compile_spec:
-        if not all([isinstance(i, torch.Tensor) or isinstance(i, Input) for i in compile_spec["inputs"]]):
-            raise KeyError("Input specs should be either torch_tensorrt.Input or torch.Tensor, found types: {}".format(
-                [type(i) for i in compile_spec["inputs"]]))
 
-        inputs = [Input._from_tensor(i) if isinstance(i, torch.Tensor) else i for i in compile_spec["inputs"]]
+def _parse_compile_spec(compile_spec_: Dict[str, Any]) -> _ts_C.CompileSpec:
+    # TODO: Remove deep copy once collections does not need partial compilation
+    compile_spec = deepcopy(compile_spec_)
+    info = _ts_C.CompileSpec()
+
+    if len(compile_spec["inputs"]) > 0:
+        if not all(
+            [
+                isinstance(i, torch.Tensor) or isinstance(i, Input)
+                for i in compile_spec["inputs"]
+            ]
+        ):
+            raise KeyError(
+                "Input specs should be either torch_tensorrt.Input or torch.Tensor, found types: {}".format(
+                    [type(i) for i in compile_spec["inputs"]]
+                )
+            )
+
+        inputs = [
+            Input._from_tensor(i) if isinstance(i, torch.Tensor) else i
+            for i in compile_spec["inputs"]
+        ]
         info.inputs = [i._to_internal() for i in inputs]
 
-    assert (len(info.inputs) > 0), "Require at least one input definition to compile model"
+    elif compile_spec["input_signature"] is not None:
+        log(
+            Level.Warning,
+            "Input signature parsing is an experimental feature, behavior and APIs may change",
+        )
+        signature = _parse_input_signature(compile_spec["input_signature"])
+        info.input_signature = _C.InputSignature(signature)  # py_object
+
+        if not compile_spec["torch_fallback"]["enabled"]:
+            raise ValueError(
+                "Grouped inputs currently requires partial compilation to be enabled, this restriction will be relaxed in a future release"
+            )
+
+        log(
+            Level.Debug,
+            "Grouped inputs currently requires additional settings to enable the feature",
+        )
+        log(
+            Level.Debug,
+            """Adding the following ops to torch_executed_ops:
+    - aten::__getitem__
+    - prim::ListConstruct
+    - prim::ListUnpack
+    - prim::TupleIndex
+    - prim::TupleConstruct
+    - prim::TupleUnpack
+""",
+        )
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append(
+            "aten::__getitem__"
+        )
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append(
+            "prim::ListConstruct"
+        )
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append("prim::ListUnpack")
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append("prim::TupleIndex")
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append(
+            "prim::TupleConstruct"
+        )
+        compile_spec["torch_fallback"]["forced_fallback_ops"].append(
+            "prim::TupleUnpack"
+        )
+
+    else:
+        raise KeyError(
+            'Module input definitions are requried to compile module. Provide a list of torch_tensorrt.Input keyed to "inputs" in the compile spec'
+        )
 
     if "enabled_precisions" in compile_spec:
-        info.enabled_precisions = _parse_enabled_precisions(compile_spec["enabled_precisions"])
+        info.enabled_precisions = _parse_enabled_precisions(
+            compile_spec["enabled_precisions"]
+        )
 
     if "calibrator" in compile_spec:
         info.ptq_calibrator = compile_spec["calibrator"]
@@ -550,10 +672,6 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         assert isinstance(compile_spec["capability"], _enums.EngineCapability)
         info.capability = compile_spec["capability"]
 
-    if "num_min_timing_iters" in compile_spec:
-        assert type(compile_spec["num_min_timing_iters"]) is int
-        info.num_min_timing_iters = compile_spec["num_min_timing_iters"]
-
     if "num_avg_timing_iters" in compile_spec:
         assert type(compile_spec["num_avg_timing_iters"]) is int
         info.num_avg_timing_iters = compile_spec["num_avg_timing_iters"]
@@ -562,6 +680,18 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         assert type(compile_spec["workspace_size"]) is int
         info.workspace_size = compile_spec["workspace_size"]
 
+    if "dla_sram_size" in compile_spec:
+        assert type(compile_spec["dla_sram_size"]) is int
+        info.dla_sram_size = compile_spec["dla_sram_size"]
+
+    if "dla_local_dram_size" in compile_spec:
+        assert type(compile_spec["dla_local_dram_size"]) is int
+        info.dla_local_dram_size = compile_spec["dla_local_dram_size"]
+
+    if "dla_global_dram_size" in compile_spec:
+        assert type(compile_spec["dla_global_dram_size"]) is int
+        info.dla_global_dram_size = compile_spec["dla_global_dram_size"]
+
     if "truncate_long_and_double" in compile_spec:
         assert type(compile_spec["truncate_long_and_double"]) is bool
         info.truncate_long_and_double = compile_spec["truncate_long_and_double"]
@@ -569,22 +699,29 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
     if "torch_fallback" in compile_spec:
         info.torch_fallback = _parse_torch_fallback(compile_spec["torch_fallback"])
 
+    log(Level.Debug, str(info))
+
     return info
 
 
-
    [docs]def TensorRTCompileSpec(inputs=[],
-                        device=Device._current_device(),
-                        disable_tf32=False,
-                        sparse_weights=False,
-                        enabled_precisions=set(),
-                        refit=False,
-                        debug=False,
-                        capability=_enums.EngineCapability.default,
-                        num_min_timing_iters=2,
-                        num_avg_timing_iters=1,
-                        workspace_size=0,
-                        truncate_long_and_double=False,
-                        calibrator=None) -> torch.classes.tensorrt.CompileSpec:
+
    [docs]def TensorRTCompileSpec(
+    inputs=[],
+    input_signature=None,
+    device=Device._current_device(),
+    disable_tf32=False,
+    sparse_weights=False,
+    enabled_precisions=set(),
+    refit=False,
+    debug=False,
+    capability=_enums.EngineCapability.default,
+    num_avg_timing_iters=1,
+    workspace_size=0,
+    dla_sram_size=1048576,
+    dla_local_dram_size=1073741824,
+    dla_global_dram_size=536870912,
+    truncate_long_and_double=False,
+    calibrator=None,
+) -> torch.classes.tensorrt.CompileSpec:
     """Utility to create a formated spec dictionary for using the PyTorch TensorRT backend
 
     Keyword Args:
@@ -614,7 +751,6 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         refit (bool): Enable refitting
         debug (bool): Enable debuggable engine
         capability (torch_tensorrt.EngineCapability): Restrict kernel selection to safe gpu kernels or safe dla kernels
-        num_min_timing_iters (int): Number of minimization timing iterations used to select kernels
         num_avg_timing_iters (int): Number of averaging timing iterations used to select kernels
         workspace_size (int): Maximum size of workspace given to TensorRT
         truncate_long_and_double (bool): Truncate weights provided in int64 or double (float64) to int32 and float32
@@ -626,34 +762,34 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
 
     compile_spec = {
         "inputs": inputs,
+        # "input_signature": input_signature,
         "device": device,
-        "disable_tf32":
-            disable_tf32,  # Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
-        "sparse_weights": sparse_weights,  #Enable sparsity for convolution and fully connected layers.
+        "disable_tf32": disable_tf32,  # Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
+        "sparse_weights": sparse_weights,  # Enable sparsity for convolution and fully connected layers.
         "enabled_precisions": enabled_precisions,  # Enabling FP16 kernels
         "refit": refit,  # enable refit
         "debug": debug,  # enable debuggable engine
         "capability": capability,  # Restrict kernel selection to safe gpu kernels or safe dla kernels
-        "num_min_timing_iters": num_min_timing_iters,  # Number of minimization timing iterations used to select kernels
         "num_avg_timing_iters": num_avg_timing_iters,  # Number of averaging timing iterations used to select kernels
         "workspace_size": workspace_size,  # Maximum size of workspace given to TensorRT
+        "dla_sram_size": dla_sram_size,  # Fast software managed RAM used by DLA to communicate within a layer.
+        "dla_local_dram_size": dla_local_dram_size,  # Host RAM used by DLA to share intermediate tensor data across operations
+        "dla_global_dram_size": dla_global_dram_size,  # Host RAM used by DLA to store weights and metadata for execution
         "calibrator": calibrator,
-        "truncate_long_and_double": truncate_long_and_double
+        "truncate_long_and_double": truncate_long_and_double,
     }
 
     parsed_spec = _parse_compile_spec(compile_spec)
 
     backend_spec = torch.classes.tensorrt.CompileSpec()
 
+    if input_signature is not None:
+        raise ValueError(
+            "Input signature parsing is not currently supported in the TorchScript backend integration"
+        )
+
     for i in parsed_spec.inputs:
-        clone = torch.classes.tensorrt._Input()
-        clone._set_min(i.min)
-        clone._set_opt(i.opt)
-        clone._set_max(i.max)
-        clone._set_dtype(i.dtype)
-        clone._set_format(i.format)
-        clone._set_input_is_dynamic(i.input_is_dynamic)
-        clone._set_explicit_set_dtype(i._explicit_set_dtype)
+        clone = _internal_input_to_torch_class_input(i)
         backend_spec._append_input(clone)
 
     d = torch.classes.tensorrt._Device()
@@ -670,8 +806,12 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
     torch_fallback = torch.classes.tensorrt._TorchFallback()
     torch_fallback._set_enabled(parsed_spec.torch_fallback.enabled)
     torch_fallback._set_min_block_size(parsed_spec.torch_fallback.min_block_size)
-    torch_fallback._set_forced_fallback_operators(parsed_spec.torch_fallback.forced_fallback_operators)
-    torch_fallback._set_forced_fallback_modules(parsed_spec.torch_fallback.forced_fallback_modules)
+    torch_fallback._set_forced_fallback_operators(
+        parsed_spec.torch_fallback.forced_fallback_operators
+    )
+    torch_fallback._set_forced_fallback_modules(
+        parsed_spec.torch_fallback.forced_fallback_modules
+    )
 
     backend_spec._set_device(d)
     backend_spec._set_torch_fallback(torch_fallback)
@@ -682,9 +822,11 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
     backend_spec._set_debug(parsed_spec.debug)
     backend_spec._set_refit(parsed_spec.refit)
     backend_spec._set_capability(int(parsed_spec.capability))
-    backend_spec._set_num_min_timing_iters(parsed_spec.num_min_timing_iters)
     backend_spec._set_num_avg_timing_iters(parsed_spec.num_avg_timing_iters)
     backend_spec._set_workspace_size(parsed_spec.workspace_size)
+    backend_spec._set_dla_sram_size(parsed_spec.dla_sram_size)
+    backend_spec._set_dla_local_dram_size(parsed_spec.dla_local_dram_size)
+    backend_spec._set_dla_global_dram_size(parsed_spec.dla_global_dram_size)
     backend_spec._set_truncate_long_and_double(parsed_spec.truncate_long_and_double)
     backend_spec._set_ptq_calibrator(parsed_spec._get_calibrator_handle())
 
@@ -692,16 +834,16 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
 
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -709,11 +851,11 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
 
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -723,19 +865,19 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
         
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -744,9 +886,9 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
          
          
          
+     
 
-
-
+  
 
   
   
@@ -757,7 +899,7 @@ 
    Source code for torch_tensorrt.ts._compile_spec
    <
       jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_modules/torch_tensorrt/ts/_compiler.html b/docs/_modules/torch_tensorrt/ts/_compiler.html
index 4731a41f5d..727225f701 100644
--- a/docs/_modules/torch_tensorrt/ts/_compiler.html
+++ b/docs/_modules/torch_tensorrt/ts/_compiler.html
@@ -6,24 +6,24 @@
   
 
   
-
+  
   
-
+  
   torch_tensorrt.ts._compiler — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,13 +335,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    Source code for torch_tensorrt.ts._compiler
     from typing import List, Dict, Any
 import torch
@@ -356,24 +354,29 @@ 
    Source code for torch_tensorrt.ts._compiler
     from types import FunctionType
 
 
-
    [docs]def compile(module: torch.jit.ScriptModule,
-            inputs=[],
-            device=Device._current_device(),
-            disable_tf32=False,
-            sparse_weights=False,
-            enabled_precisions=set(),
-            refit=False,
-            debug=False,
-            capability=_enums.EngineCapability.default,
-            num_min_timing_iters=2,
-            num_avg_timing_iters=1,
-            workspace_size=0,
-            calibrator=None,
-            truncate_long_and_double=False,
-            require_full_compilation=False,
-            min_block_size=3,
-            torch_executed_ops=[],
-            torch_executed_modules=[]) -> torch.jit.ScriptModule:
+
    [docs]def compile(
+    module: torch.jit.ScriptModule,
+    inputs=[],
+    input_signature=None,
+    device=Device._current_device(),
+    disable_tf32=False,
+    sparse_weights=False,
+    enabled_precisions=set(),
+    refit=False,
+    debug=False,
+    capability=_enums.EngineCapability.default,
+    num_avg_timing_iters=1,
+    workspace_size=0,
+    dla_sram_size=1048576,
+    dla_local_dram_size=1073741824,
+    dla_global_dram_size=536870912,
+    calibrator=None,
+    truncate_long_and_double=False,
+    require_full_compilation=False,
+    min_block_size=3,
+    torch_executed_ops=[],
+    torch_executed_modules=[],
+) -> torch.jit.ScriptModule:
     """Compile a TorchScript module for NVIDIA GPUs using TensorRT
 
     Takes a existing TorchScript module and a set of settings to configure the compiler
@@ -402,6 +405,19 @@ 
    Source code for torch_tensorrt.ts._compiler
                         torch.randn((1, 3, 224, 244)) # Use an example tensor and let torch_tensorrt infer settings
                 ]
 
+        input_signature Union(List, Tuple, torch_tensorrt.Input, torch.Tensor): A formatted collection of input specifications for the module. Input Sizes can be specified as torch sizes, tuples or lists. dtypes can be specified using
+            torch datatypes or torch_tensorrt datatypes and you can use either torch devices or the torch_tensorrt device type enum to select device type. **This API should be considered beta-level stable and may change in the future** ::
+
+                input_signature=([
+                    torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+                    torch_tensorrt.Input(
+                        min_shape=(1, 224, 224, 3),
+                        opt_shape=(1, 512, 512, 3),
+                        max_shape=(1, 1024, 1024, 3),
+                        dtype=torch.int32
+                        format=torch.channel_last
+                    ), # Dynamic input shape for input #2
+                ], torch.randn((1, 3, 224, 244))) # Use an example tensor and let torch_tensorrt infer settings for input #3
         device (Union(torch_tensorrt.Device, torch.device, dict)): Target device for TensorRT engines to run on ::
 
             device=torch_tensorrt.Device("dla:1", allow_gpu_fallback=True)
@@ -412,9 +428,11 @@ 
    Source code for torch_tensorrt.ts._compiler
             refit (bool): Enable refitting
         debug (bool): Enable debuggable engine
         capability (torch_tensorrt.EngineCapability): Restrict kernel selection to safe gpu kernels or safe dla kernels
-        num_min_timing_iters (int): Number of minimization timing iterations used to select kernels
         num_avg_timing_iters (int): Number of averaging timing iterations used to select kernels
         workspace_size (int): Maximum size of workspace given to TensorRT
+        dla_sram_size (int): Fast software managed RAM used by DLA to communicate within a layer.
+        dla_local_dram_size (int): Host RAM used by DLA to share intermediate tensor data across operations
+        dla_global_dram_size (int): Host RAM used by DLA to store weights and metadata for execution
         truncate_long_and_double (bool): Truncate weights provided in int64 or double (float64) to int32 and float32
         calibrator (Union(torch_tensorrt._C.IInt8Calibrator, tensorrt.IInt8Calibrator)): Calibrator object which will provide data to the PTQ system for INT8 Calibration
         require_full_compilation (bool): Require modules to be compiled end to end or return an error as opposed to returning a hybrid graph where operations that cannot be run in TensorRT are run in PyTorch
@@ -428,23 +446,26 @@ 
    Source code for torch_tensorrt.ts._compiler
     
     if isinstance(module, torch.jit.ScriptFunction):
         raise TypeError(
-            "torch.jit.ScriptFunction currently is not directly supported, wrap the function in a module to compile")
+            "torch.jit.ScriptFunction currently is not directly supported, wrap the function in a module to compile"
+        )
 
-    if require_full_compilation and (len(torch_executed_modules) > 0 or len(torch_executed_ops) > 0):
+    if require_full_compilation and (
+        len(torch_executed_modules) > 0 or len(torch_executed_ops) > 0
+    ):
         raise ValueError(
-            "require_full_compilation is enabled however the list of modules and ops to run in torch is not empty. Found: torch_executed_ops: "
-            + torch_executed_ops + ", torch_executed_modules: " + torch_executed_modules)
+            f"require_full_compilation is enabled however the list of modules and ops to run in torch is not empty. Found: torch_executed_ops: {torch_executed_ops}, torch_executed_modules: {torch_executed_modules}"
+        )
 
     spec = {
         "inputs": inputs,
+        "input_signature": input_signature,
         "device": device,
         "disable_tf32": disable_tf32,  # Force FP32 layers to use traditional as FP32 format
-        "sparse_weights": sparse_weights,  #Enable sparsity for convolution and fully connected layers.
+        "sparse_weights": sparse_weights,  # Enable sparsity for convolution and fully connected layers.
         "enabled_precisions": enabled_precisions,  # Enabling FP16 kernels
         "refit": refit,  # enable refit
         "debug": debug,  # enable debuggable engine
         "capability": capability,  # Restrict kernel selection to safe gpu kernels or safe dla kernels
-        "num_min_timing_iters": num_min_timing_iters,  # Number of minimization timing iterations used to select kernels
         "num_avg_timing_iters": num_avg_timing_iters,  # Number of averaging timing iterations used to select kernels
         "workspace_size": workspace_size,  # Maximum size of workspace given to TensorRT
         "calibrator": calibrator,
@@ -453,8 +474,8 @@ 
    Source code for torch_tensorrt.ts._compiler
                 "enabled": not require_full_compilation,
             "forced_fallback_ops": torch_executed_ops,
             "forced_fallback_modules": torch_executed_modules,
-            "min_block_size": min_block_size
-        }
+            "min_block_size": min_block_size,
+        },
     }
 
     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
@@ -462,21 +483,25 @@ 
    Source code for torch_tensorrt.ts._compiler
         return compiled_module
    
 
 
-
    [docs]def convert_method_to_trt_engine(module: torch.jit.ScriptModule,
-                                 method_name: str,
-                                 inputs=[],
-                                 device=Device._current_device(),
-                                 disable_tf32=False,
-                                 sparse_weights=False,
-                                 enabled_precisions=set(),
-                                 refit=False,
-                                 debug=False,
-                                 capability=_enums.EngineCapability.default,
-                                 num_min_timing_iters=2,
-                                 num_avg_timing_iters=1,
-                                 workspace_size=0,
-                                 truncate_long_and_double=False,
-                                 calibrator=None) -> str:
+
    [docs]def convert_method_to_trt_engine(
+    module: torch.jit.ScriptModule,
+    method_name: str,
+    inputs=[],
+    device=Device._current_device(),
+    disable_tf32=False,
+    sparse_weights=False,
+    enabled_precisions=set(),
+    refit=False,
+    debug=False,
+    capability=_enums.EngineCapability.default,
+    num_avg_timing_iters=1,
+    workspace_size=0,
+    dla_sram_size=1048576,
+    dla_local_dram_size=1073741824,
+    dla_global_dram_size=536870912,
+    truncate_long_and_double=False,
+    calibrator=None,
+) -> str:
     """Convert a TorchScript module method to a serialized TensorRT engine
 
     Converts a specified method of a module to a serialized TensorRT engine given a dictionary of conversion settings
@@ -503,6 +528,20 @@ 
    Source code for torch_tensorrt.ts._compiler
                         torch.randn((1, 3, 224, 244)) # Use an example tensor and let torch_tensorrt infer settings
                 ]
 
+        input_signature Union(List, Tuple, torch_tensorrt.Input, torch.Tensor): A formatted collection of input specifications for the module. Input Sizes can be specified as torch sizes, tuples or lists. dtypes can be specified using
+            torch datatypes or torch_tensorrt datatypes and you can use either torch devices or the torch_tensorrt device type enum to select device type. **This API should be considered beta-level stable and may change in the future** ::
+
+                input_signature=([
+                    torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+                    torch_tensorrt.Input(
+                        min_shape=(1, 224, 224, 3),
+                        opt_shape=(1, 512, 512, 3),
+                        max_shape=(1, 1024, 1024, 3),
+                        dtype=torch.int32
+                        format=torch.channel_last
+                    ), # Dynamic input shape for input #2
+                ], torch.randn((1, 3, 224, 244))) # Use an example tensor and let torch_tensorrt infer settings for input #3
+
         device (Union(torch_tensorrt.Device, torch.device, dict)): Target device for TensorRT engines to run on ::
 
             device=torch_tensorrt.Device("dla:1", allow_gpu_fallback=True)
@@ -513,9 +552,11 @@ 
    Source code for torch_tensorrt.ts._compiler
             refit (bool): Enable refitting
         debug (bool): Enable debuggable engine
         capability (torch_tensorrt.EngineCapability): Restrict kernel selection to safe gpu kernels or safe dla kernels
-        num_min_timing_iters (int): Number of minimization timing iterations used to select kernels
         num_avg_timing_iters (int): Number of averaging timing iterations used to select kernels
         workspace_size (int): Maximum size of workspace given to TensorRT
+        dla_sram_size (int): Fast software managed RAM used by DLA to communicate within a layer.
+        dla_local_dram_size (int): Host RAM used by DLA to share intermediate tensor data across operations
+        dla_global_dram_size (int): Host RAM used by DLA to store weights and metadata for execution
         truncate_long_and_double (bool): Truncate weights provided in int64 or double (float64) to int32 and float32
         calibrator (Union(torch_tensorrt._C.IInt8Calibrator, tensorrt.IInt8Calibrator)): Calibrator object which will provide data to the PTQ system for INT8 Calibration
 
@@ -524,29 +565,32 @@ 
    Source code for torch_tensorrt.ts._compiler
         """
     if isinstance(module, torch.jit.ScriptFunction):
         raise TypeError(
-            "torch.jit.ScriptFunctions currently are not directly supported, wrap the function in a module to compile")
+            "torch.jit.ScriptFunctions currently are not directly supported, wrap the function in a module to compile"
+        )
 
     compile_spec = {
         "inputs": inputs,
         "device": device,
-        "disable_tf32":
-            disable_tf32,  # Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
-        "sparse_weights": sparse_weights,  #Enable sparsity for convolution and fully connected layers.
+        "disable_tf32": disable_tf32,  # Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
+        "sparse_weights": sparse_weights,  # Enable sparsity for convolution and fully connected layers.
         "enabled_precisions": enabled_precisions,  # Enabling FP16 kernels
         "refit": refit,  # enable refit
         "debug": debug,  # enable debuggable engine
         "capability": capability,  # Restrict kernel selection to safe gpu kernels or safe dla kernels
-        "num_min_timing_iters": num_min_timing_iters,  # Number of minimization timing iterations used to select kernels
         "num_avg_timing_iters": num_avg_timing_iters,  # Number of averaging timing iterations used to select kernels
         "workspace_size": workspace_size,  # Maximum size of workspace given to TensorRT
         "calibrator": calibrator,
-        "truncate_long_and_double": truncate_long_and_double
+        "truncate_long_and_double": truncate_long_and_double,
     }
 
-    return _C.convert_graph_to_trt_engine(module._c, method_name, _parse_compile_spec(compile_spec))
    
+    return _C.convert_graph_to_trt_engine(
+        module._c, method_name, _parse_compile_spec(compile_spec)
+    )
    
 
 
-
    [docs]def embed_engine_in_new_module(serialized_engine: bytes, device=Device._current_device()) -> torch.jit.ScriptModule:
+
    [docs]def embed_engine_in_new_module(
+    serialized_engine: bytes, device=Device._current_device()
+) -> torch.jit.ScriptModule:
     """Takes a pre-built serialized TensorRT engine and embeds it within a TorchScript module
 
     Takes a pre-built serialied TensorRT engine (as bytes) and embeds it within a TorchScript module.
@@ -592,16 +636,16 @@ 
    Source code for torch_tensorrt.ts._compiler
     
    
 
              
    
-
+             
             
    
             
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -609,11 +653,11 @@ 
    Source code for torch_tensorrt.ts._compiler
     
     
    
   
    
-
+    
       
    
         Built with Sphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
    
 
@@ -623,19 +667,19 @@ 
    Source code for torch_tensorrt.ts._compiler
             
    
           
    
             
    
-
+              
             
    
           
    
         
    
       
    
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -644,9 +688,9 @@ 
    Source code for torch_tensorrt.ts._compiler
              
          
          
+     
 
-
-
+  
 
   
   
@@ -657,7 +701,7 @@ 
    Source code for torch_tensorrt.ts._compiler
           jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
-  
+   
 
   
 
diff --git a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst.txt b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst.txt
index 326995993e..61447e1ada 100644
--- a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst.txt
+++ b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst.txt
@@ -33,6 +33,8 @@ Included By
 
 - :ref:`file_cpp_include_torch_tensorrt_logging.h`
 
+- :ref:`file_cpp_include_torch_tensorrt_ptq.h`
+
 - :ref:`file_cpp_include_torch_tensorrt_torch_tensorrt.h`
 
 
diff --git a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst.txt b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst.txt
index 074de6eb5e..34b64af6e0 100644
--- a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst.txt
+++ b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst.txt
@@ -47,6 +47,8 @@ Includes
 
 - ``torch_tensorrt/logging.h`` (:ref:`file_cpp_include_torch_tensorrt_logging.h`)
 
+- ``torch_tensorrt/macros.h`` (:ref:`file_cpp_include_torch_tensorrt_macros.h`)
+
 - ``vector``
 
 
@@ -76,7 +78,7 @@ Functions
 ---------
 
 
-- :ref:`exhale_function_ptq_8h_1a9835f6e605dce1abf442a55b64d6dffa`
+- :ref:`exhale_function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c`
 
-- :ref:`exhale_function_ptq_8h_1a83ff2be7e0b80bc7434de415861dc039`
+- :ref:`exhale_function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178`
 
diff --git a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
index a81343d773..e29afa26a3 100644
--- a/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
+++ b/docs/_sources/_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
@@ -39,6 +39,8 @@ Includes
 
 - ``string``
 
+- ``torch/custom_class.h``
+
 - ``torch_tensorrt/macros.h`` (:ref:`file_cpp_include_torch_tensorrt_macros.h`)
 
 - ``vector``
@@ -63,6 +65,8 @@ Classes
 
 - :ref:`exhale_struct_structtorch__tensorrt_1_1Device`
 
+- :ref:`exhale_struct_structtorch__tensorrt_1_1GraphInputs`
+
 - :ref:`exhale_struct_structtorch__tensorrt_1_1Input`
 
 - :ref:`exhale_struct_structtorch__tensorrt_1_1torchscript_1_1CompileSpec`
diff --git a/docs/_sources/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.rst.txt b/docs/_sources/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.rst.txt
new file mode 100644
index 0000000000..5d9188270f
--- /dev/null
+++ b/docs/_sources/_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.rst.txt
@@ -0,0 +1,14 @@
+.. _exhale_function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c:
+
+Template Function torch_tensorrt::ptq::make_int8_cache_calibrator
+=================================================================
+
+- Defined in :ref:`file_cpp_include_torch_tensorrt_ptq.h`
+
+
+Function Documentation
+----------------------
+
+
+.. doxygenfunction:: torch_tensorrt::ptq::make_int8_cache_calibrator(const std::string&)
+   :project: Torch-TensorRT
\ No newline at end of file
diff --git a/docs/_sources/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.rst.txt b/docs/_sources/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.rst.txt
new file mode 100644
index 0000000000..9551f48bb6
--- /dev/null
+++ b/docs/_sources/_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.rst.txt
@@ -0,0 +1,14 @@
+.. _exhale_function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178:
+
+Template Function torch_tensorrt::ptq::make_int8_calibrator
+===========================================================
+
+- Defined in :ref:`file_cpp_include_torch_tensorrt_ptq.h`
+
+
+Function Documentation
+----------------------
+
+
+.. doxygenfunction:: torch_tensorrt::ptq::make_int8_calibrator(DataLoader, const std::string&, bool)
+   :project: Torch-TensorRT
\ No newline at end of file
diff --git a/docs/_sources/_cpp_api/namespace_torch_tensorrt.rst.txt b/docs/_sources/_cpp_api/namespace_torch_tensorrt.rst.txt
index 94e153b999..43a0afb20d 100644
--- a/docs/_sources/_cpp_api/namespace_torch_tensorrt.rst.txt
+++ b/docs/_sources/_cpp_api/namespace_torch_tensorrt.rst.txt
@@ -30,6 +30,8 @@ Classes
 
 - :ref:`exhale_struct_structtorch__tensorrt_1_1Device`
 
+- :ref:`exhale_struct_structtorch__tensorrt_1_1GraphInputs`
+
 - :ref:`exhale_struct_structtorch__tensorrt_1_1Input`
 
 - :ref:`exhale_class_classtorch__tensorrt_1_1DataType`
diff --git a/docs/_sources/_cpp_api/namespace_torch_tensorrt__ptq.rst.txt b/docs/_sources/_cpp_api/namespace_torch_tensorrt__ptq.rst.txt
index b20546a725..bdc39cb326 100644
--- a/docs/_sources/_cpp_api/namespace_torch_tensorrt__ptq.rst.txt
+++ b/docs/_sources/_cpp_api/namespace_torch_tensorrt__ptq.rst.txt
@@ -26,6 +26,6 @@ Functions
 ---------
 
 
-- :ref:`exhale_function_ptq_8h_1a9835f6e605dce1abf442a55b64d6dffa`
+- :ref:`exhale_function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c`
 
-- :ref:`exhale_function_ptq_8h_1a83ff2be7e0b80bc7434de415861dc039`
+- :ref:`exhale_function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178`
diff --git a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst.txt b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst.txt
index af68cbb678..ec413cb1b7 100644
--- a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst.txt
+++ b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst.txt
@@ -18,10 +18,10 @@ Program Listing for File logging.h
     * LICENSE file in the root directory of this source tree.
     */
    #pragma once
-
+   
    #include 
    #include "torch_tensorrt/macros.h"
-
+   
    namespace torch_tensorrt {
    namespace logging {
    enum Level {
@@ -32,19 +32,19 @@ Program Listing for File logging.h
      kDEBUG,
      kGRAPH,
    };
-
+   
    // Are these ones necessary for the user?
    TORCHTRT_API std::string get_logging_prefix();
    TORCHTRT_API void set_logging_prefix(std::string prefix);
-
+   
    TORCHTRT_API void set_reportable_log_level(Level lvl);
-
+   
    TORCHTRT_API void set_is_colored_output_on(bool colored_output_on);
-
+   
    TORCHTRT_API Level get_reportable_log_level();
-
+   
    TORCHTRT_API bool get_is_colored_output_on();
-
+   
    // Dont know if we want this?
    TORCHTRT_API void log(Level lvl, std::string msg);
    } // namespace logging
diff --git a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst.txt b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst.txt
index 2753371798..ff11f2dd6f 100644
--- a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst.txt
+++ b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst.txt
@@ -18,7 +18,10 @@ Program Listing for File macros.h
     * LICENSE file in the root directory of this source tree.
     */
    #pragma once
-
+   
+   #if defined(USE_CMAKE_GENERATED_EXPORT_HEADER)
+   #include 
+   #else
    #if defined(__GNUC__)
    #define TORCHTRT_API __attribute__((__visibility__("default")))
    #define TORCHTRT_HIDDEN __attribute__((__visibility__("hidden")))
@@ -26,18 +29,19 @@ Program Listing for File macros.h
    #define TORCHTRT_API
    #define TORCHTRT_HIDDEN
    #endif // defined(__GNUC__)
-
+   #endif // defined(USE_CMAKE_GENERATED_EXPORT_HEADER)
+   
    // Does this need to be gaurded or something?
    #define XSTR(x) #x
    #define STR(x) XSTR(x)
-
+   
    #define TORCH_TENSORRT_MAJOR_VERSION 1
    #define TORCH_TENSORRT_MINOR_VERSION 2
    #define TORCH_TENSORRT_PATCH_VERSION 0
    #define TORCH_TENSORRT_VERSION      \
      STR(TORCH_TENSORRT_MAJOR_VERSION) \
      "." STR(TORCH_TENSORRT_MINOR_VERSION) "." STR(TORCH_TENSORRT_PATCH_VERSION)
-
+   
    // Setup namespace aliases for ease of use
    namespace torch_tensorrt {
    namespace torchscript {}
diff --git a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst.txt b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst.txt
index 12c51674f3..8f4e8e4865 100644
--- a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst.txt
+++ b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst.txt
@@ -18,7 +18,7 @@ Program Listing for File ptq.h
     * LICENSE file in the root directory of this source tree.
     */
    #pragma once
-
+   
    #include 
    #include 
    #include 
@@ -26,32 +26,33 @@ Program Listing for File ptq.h
    #include 
    #include 
    #include 
-
+   
    #include "NvInfer.h"
    #include "torch/torch.h"
    #include "torch_tensorrt/logging.h"
-
+   #include "torch_tensorrt/macros.h"
+   
    #ifndef DOXYGEN_SHOULD_SKIP_THIS
    namespace nvinfer1 {
    class IInt8Calibrator;
    class IInt8EntropyCalibrator2;
    } // namespace nvinfer1
-
+   
    namespace torch_tensorrt {
    namespace ptq {
-   bool get_batch_impl(void* bindings[], const char* names[], int nbBindings, torch::Tensor& data);
+   TORCHTRT_API bool get_batch_impl(void* bindings[], const char* names[], int nbBindings, torch::Tensor& data);
    }
    } // namespace torch_tensorrt
    #endif // DOXYGEN_SHOULD_SKIP_THIS
-
+   
    namespace torch_tensorrt {
    namespace ptq {
-
+   
    template 
    class Int8Calibrator : Algorithm {
      using DataLoader = typename DataLoaderUniquePtr::element_type;
      using Batch = typename DataLoader::super::BatchType;
-
+   
     public:
      Int8Calibrator(DataLoaderUniquePtr dataloader, const std::string& cache_file_path, bool use_cache)
          : dataloader_(dataloader.get()), cache_file_path_(cache_file_path), use_cache_(use_cache) {
@@ -60,7 +61,7 @@ Program Listing for File ptq.h
        }
        it_ = batched_data_.begin();
      }
-
+   
      int getBatchSize() const noexcept override {
        // HACK: Torch-TensorRT only uses explict batch sizing, INT8 Calibrator does not
        // work when reporting the batch size here and having explicity batching.
@@ -68,7 +69,7 @@ Program Listing for File ptq.h
        return 1;
        // return static_cast(dataloader_->options().batch_size);
      }
-
+   
      bool getBatch(void* bindings[], const char* names[], int nbBindings) noexcept override {
        if (it_ != batched_data_.end()) {
          auto status = get_batch_impl(bindings, names, nbBindings, *it_);
@@ -80,13 +81,13 @@ Program Listing for File ptq.h
          return false;
        }
      }
-
+   
      const void* readCalibrationCache(size_t& length) noexcept override {
        if (use_cache_) {
          std::stringstream ss;
          ss << "Reading Calibration Cache from " << cache_file_path_;
          logging::log(logging::Level::kINFO, ss.str());
-
+   
          cache_.clear();
          std::ifstream input(cache_file_path_, std::ios::binary);
          input >> std::noskipws;
@@ -99,7 +100,7 @@ Program Listing for File ptq.h
        }
        return nullptr;
      }
-
+   
      void writeCalibrationCache(const void* cache, size_t length) noexcept override {
        std::ofstream cache_file(cache_file_path_, std::ios::binary);
        cache_file.write(reinterpret_cast(cache), length);
@@ -107,11 +108,11 @@ Program Listing for File ptq.h
        ss << "Saved Calibration Cache to " << cache_file_path_;
        logging::log(logging::Level::kINFO, ss.str());
      }
-
+   
      operator nvinfer1::IInt8Calibrator*() {
        return reinterpret_cast(this);
      }
-
+   
     private:
      DataLoader* dataloader_;
      const std::string& cache_file_path_;
@@ -121,28 +122,28 @@ Program Listing for File ptq.h
      std::vector batched_data_;
      std::vector::iterator it_;
    };
-
+   
    template 
    class Int8CacheCalibrator : Algorithm {
     public:
      Int8CacheCalibrator(const std::string& cache_file_path) : cache_file_path_(cache_file_path) {}
-
+   
      int getBatchSize() const noexcept override {
        // HACK: Torch-TensorRT only uses explict batch sizing, INT8 Calibrator does not
        // work when reporting the batch size here and having explicity batching.
        // So we just report batch size 1 (warnings will still be printed out).
        return 1;
      }
-
+   
      bool getBatch(void* bindings[], const char* names[], int nbBindings) noexcept override {
        return false;
      }
-
+   
      const void* readCalibrationCache(size_t& length) noexcept override {
        std::stringstream ss;
        ss << "Reading Calibration Cache from " << cache_file_path_;
        logging::log(logging::Level::kINFO, ss.str());
-
+   
        cache_.clear();
        std::ifstream input(cache_file_path_, std::ios::binary);
        input >> std::noskipws;
@@ -153,7 +154,7 @@ Program Listing for File ptq.h
        length = cache_.size();
        return length ? cache_.data() : nullptr;
      }
-
+   
      void writeCalibrationCache(const void* cache, size_t length) noexcept override {
        std::ofstream cache_file(cache_file_path_, std::ios::binary);
        cache_file.write(reinterpret_cast(cache), length);
@@ -161,29 +162,29 @@ Program Listing for File ptq.h
        ss << "Saved Calibration Cache to " << cache_file_path_;
        logging::log(logging::Level::kINFO, ss.str());
      }
-
+   
      operator nvinfer1::IInt8Calibrator*() {
        return reinterpret_cast(this);
      }
-
+   
     private:
      const std::string& cache_file_path_;
      size_t cache_size_ = 0;
      std::vector cache_;
    };
-
+   
    template 
-   TORCHTRT_API inline Int8Calibrator make_int8_calibrator(
+   inline Int8Calibrator make_int8_calibrator(
        DataLoader dataloader,
        const std::string& cache_file_path,
        bool use_cache) {
      return Int8Calibrator(std::move(dataloader), cache_file_path, use_cache);
    }
-
+   
    template 
-   TORCHTRT_API inline Int8CacheCalibrator make_int8_cache_calibrator(const std::string& cache_file_path) {
+   inline Int8CacheCalibrator make_int8_cache_calibrator(const std::string& cache_file_path) {
      return Int8CacheCalibrator(cache_file_path);
    }
-
+   
    } // namespace ptq
    } // namespace torch_tensorrt
diff --git a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
index 500f4fb2e4..7e00b9350a 100644
--- a/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
+++ b/docs/_sources/_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst.txt
@@ -17,16 +17,19 @@ Program Listing for File torch_tensorrt.h
     * This library is licensed under the BSD-style license found in the
     * LICENSE file in the root directory of this source tree.
     */
-
+   
    #pragma once
-
+   
    #include 
    #include 
    #include 
    #include 
    #include 
    #include 
-
+   #include "torch/custom_class.h"
+   
+   #include "torch_tensorrt/macros.h"
+   
    // Just include the .h?
    #ifndef DOXYGEN_SHOULD_SKIP_THIS
    namespace torch {
@@ -35,22 +38,21 @@ Program Listing for File torch_tensorrt.h
    struct Module;
    } // namespace jit
    } // namespace torch
-
+   
    namespace c10 {
    enum class DeviceType : int8_t;
    enum class ScalarType : int8_t;
    template 
    class ArrayRef;
    } // namespace c10
-
+   
    namespace nvinfer1 {
    class IInt8Calibrator;
    }
    #endif // DOXYGEN_SHOULD_SKIP_THIS
-
-   #include "torch_tensorrt/macros.h"
+   
    namespace torch_tensorrt {
-   class TORCHTRT_API DataType {
+   class DataType {
     public:
      enum Value : int8_t {
        kFloat,
@@ -60,10 +62,10 @@ Program Listing for File torch_tensorrt.h
        kBool,
        kUnknown
      };
-
+   
      DataType() = default;
      constexpr DataType(Value t) : value(t) {}
-     DataType(c10::ScalarType t);
+     TORCHTRT_API DataType(c10::ScalarType t);
      operator Value() const {
        return value;
      }
@@ -80,12 +82,12 @@ Program Listing for File torch_tensorrt.h
      constexpr bool operator!=(DataType::Value other) const {
        return value != other;
      }
-
+   
     private:
-     friend std::ostream& operator<<(std::ostream& os, const DataType& dtype);
+     friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const DataType& dtype);
      Value value;
    };
-
+   
    struct Device {
      class DeviceType {
       public:
@@ -93,7 +95,7 @@ Program Listing for File torch_tensorrt.h
          kGPU,
          kDLA,
        };
-
+   
        DeviceType() = default;
        constexpr DeviceType(Value t) : value(t) {}
        DeviceType(c10::DeviceType t);
@@ -107,45 +109,45 @@ Program Listing for File torch_tensorrt.h
        constexpr bool operator!=(DeviceType other) const {
          return value != other.value;
        }
-
+   
       private:
        Value value;
      };
-
+   
      DeviceType device_type;
-
+   
      /*
       * Target gpu id
       */
      int64_t gpu_id;
-
+   
      /*
       * When using DLA core on NVIDIA AGX platforms gpu_id should be set as Xavier device
       */
      int64_t dla_core;
-
+   
      bool allow_gpu_fallback;
-
+   
      Device() : device_type(DeviceType::kGPU), gpu_id(0), dla_core(0), allow_gpu_fallback(false) {}
    };
-
+   
    enum class EngineCapability : int8_t {
      kSTANDARD,
      kSAFETY,
      kDLA_STANDALONE,
    };
-
-   class TORCHTRT_API TensorFormat {
+   
+   class TensorFormat {
     public:
      enum Value : int8_t {
        kContiguous,
        kChannelsLast,
        kUnknown,
      };
-
+   
      TensorFormat() = default;
      constexpr TensorFormat(Value t) : value(t) {}
-     TensorFormat(at::MemoryFormat t);
+     TORCHTRT_API TensorFormat(at::MemoryFormat t);
      operator Value() const {
        return value;
      }
@@ -162,120 +164,131 @@ Program Listing for File torch_tensorrt.h
      constexpr bool operator!=(TensorFormat::Value other) const {
        return value != other;
      }
-
+   
     private:
-     friend std::ostream& operator<<(std::ostream& os, const TensorFormat& format);
+     friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const TensorFormat& format);
      Value value;
    };
-
-   struct TORCHTRT_API Input {
+   
+   struct TORCHTRT_API Input : torch::CustomClassHolder {
      std::vector min_shape;
      std::vector opt_shape;
      std::vector max_shape;
      std::vector shape;
      DataType dtype;
      TensorFormat format;
-
-     Input(std::vector shape, TensorFormat format = TensorFormat::kContiguous);
-
-     Input(std::vector shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous);
-
-     Input(c10::ArrayRef shape, TensorFormat format = TensorFormat::kContiguous);
-
-     Input(c10::ArrayRef shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous);
-
-     Input(
+   
+     Input() {}
+     TORCHTRT_API Input(std::vector shape, TensorFormat format = TensorFormat::kContiguous);
+   
+     TORCHTRT_API Input(std::vector shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous);
+   
+     TORCHTRT_API Input(c10::ArrayRef shape, TensorFormat format = TensorFormat::kContiguous);
+   
+     TORCHTRT_API Input(c10::ArrayRef shape, DataType dtype, TensorFormat format = TensorFormat::kContiguous);
+   
+     TORCHTRT_API Input(
          std::vector min_shape,
          std::vector opt_shape,
          std::vector max_shape,
          TensorFormat format = TensorFormat::kContiguous);
-
-     Input(
+   
+     TORCHTRT_API Input(
          std::vector min_shape,
          std::vector opt_shape,
          std::vector max_shape,
          DataType dtype,
          TensorFormat format = TensorFormat::kContiguous);
-
-     Input(
+   
+     TORCHTRT_API Input(
          c10::ArrayRef min_shape,
          c10::ArrayRef opt_shape,
          c10::ArrayRef max_shape,
          TensorFormat format = TensorFormat::kContiguous);
-
-     Input(
+   
+     TORCHTRT_API Input(
          c10::ArrayRef min_shape,
          c10::ArrayRef opt_shape,
          c10::ArrayRef max_shape,
          DataType dtype,
          TensorFormat format = TensorFormat::kContiguous);
-
-     Input(at::Tensor tensor);
-
+   
+     TORCHTRT_API Input(at::Tensor tensor);
+   
     private:
-     friend std::ostream& operator<<(std::ostream& os, const Input& input);
+     friend TORCHTRT_API std::ostream& operator<<(std::ostream& os, const Input& input);
      bool input_is_dynamic;
    };
-
+   
+   struct TORCHTRT_API GraphInputs {
+     torch::jit::IValue input_signature; // nested Input, full input spec
+     std::vector inputs; // flatten input spec
+   };
+   
    TORCHTRT_API std::string get_build_info();
-
+   
    TORCHTRT_API void dump_build_info();
-
+   
    TORCHTRT_API void set_device(const int gpu_id);
-
+   
    namespace torchscript {
-   struct TORCHTRT_API CompileSpec {
-     CompileSpec(std::vector> fixed_sizes);
-
-     CompileSpec(std::vector> fixed_sizes);
-
-     CompileSpec(std::vector inputs) : inputs(std::move(inputs)) {}
-
+   struct CompileSpec {
+     TORCHTRT_API CompileSpec(std::vector> fixed_sizes);
+   
+     TORCHTRT_API CompileSpec(std::vector> fixed_sizes);
+   
+     CompileSpec(std::vector inputs);
+   
+     CompileSpec(torch::jit::IValue input_signature);
      // Defaults should reflect TensorRT defaults for BuilderConfig
-
-     std::vector inputs;
-
+   
+     GraphInputs graph_inputs;
      std::set enabled_precisions = {DataType::kFloat};
-
+   
      bool disable_tf32 = false;
-
+   
      bool sparse_weights = false;
-
+   
      bool refit = false;
-
+   
      bool debug = false;
-
+   
      bool truncate_long_and_double = false;
-
+   
      Device device;
-
+   
      EngineCapability capability = EngineCapability::kSTANDARD;
-
-     uint64_t num_min_timing_iters = 2;
+   
      uint64_t num_avg_timing_iters = 1;
-
+   
      uint64_t workspace_size = 0;
-
+   
+     uint64_t dla_sram_size = 1048576;
+   
+     uint64_t dla_local_dram_size = 1073741824;
+   
+     uint64_t dla_global_dram_size = 536870912;
+   
      nvinfer1::IInt8Calibrator* ptq_calibrator = nullptr;
-
+   
      bool require_full_compilation = false;
-
+   
      uint64_t min_block_size = 3;
-
+   
      std::vector torch_executed_ops;
-
+   
      std::vector torch_executed_modules;
    };
-
+   
    TORCHTRT_API bool check_method_operator_support(const torch::jit::Module& module, std::string method_name);
-
+   
    TORCHTRT_API torch::jit::Module compile(const torch::jit::Module& module, CompileSpec info);
-
+   
    TORCHTRT_API std::string convert_method_to_trt_engine(
        const torch::jit::Module& module,
        std::string method_name,
        CompileSpec info);
-
+   
    TORCHTRT_API torch::jit::Module embed_engine_in_new_module(const std::string& engine, Device device);
    } // namespace torchscript
    } // namespace torch_tensorrt
diff --git a/docs/_sources/_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst.txt b/docs/_sources/_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst.txt
new file mode 100644
index 0000000000..60bc3285c6
--- /dev/null
+++ b/docs/_sources/_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst.txt
@@ -0,0 +1,17 @@
+.. _exhale_struct_structtorch__tensorrt_1_1GraphInputs:
+
+Struct GraphInputs
+==================
+
+- Defined in :ref:`file_cpp_include_torch_tensorrt_torch_tensorrt.h`
+
+
+Struct Documentation
+--------------------
+
+
+.. doxygenstruct:: torch_tensorrt::GraphInputs
+   :project: Torch-TensorRT
+   :members:
+   :protected-members:
+   :undoc-members:
\ No newline at end of file
diff --git a/docs/_sources/_cpp_api/structtorch__tensorrt_1_1Input.rst.txt b/docs/_sources/_cpp_api/structtorch__tensorrt_1_1Input.rst.txt
index 21259a70d7..74c594cd94 100644
--- a/docs/_sources/_cpp_api/structtorch__tensorrt_1_1Input.rst.txt
+++ b/docs/_sources/_cpp_api/structtorch__tensorrt_1_1Input.rst.txt
@@ -6,6 +6,15 @@ Struct Input
 - Defined in :ref:`file_cpp_include_torch_tensorrt_torch_tensorrt.h`
 
 
+Inheritance Relationships
+-------------------------
+
+Base Type
+*********
+
+- ``public CustomClassHolder``
+
+
 Struct Documentation
 --------------------
 
diff --git a/docs/_sources/cli/torchtrtc.rst.txt b/docs/_sources/cli/torchtrtc.rst.txt
new file mode 100644
index 0000000000..68f599a5cd
--- /dev/null
+++ b/docs/_sources/cli/torchtrtc.rst.txt
@@ -0,0 +1,146 @@
+.. _torchtrtc:
+
+torchtrtc
+=================================
+
+``torchtrtc`` is a CLI application for using the Torch-TensorRT compiler. It serves as an easy way to compile a
+TorchScript Module with Torch-TensorRT from the command-line to quickly check support or as part of
+a deployment pipeline. All basic features of the compiler are supported including post training
+quantization (though you must already have a calibration cache file to use the PTQ feature). The compiler can
+output two formats, either a TorchScript program with the TensorRT engine embedded or
+the TensorRT engine itself as a PLAN file.
+
+All that is required to run the program after compilation is for C++ linking against ``libtorchtrt.so``
+or in Python importing the torch_tensorrt package. All other aspects of using compiled modules are identical
+to standard TorchScript. Load with ``torch.jit.load()`` and run like you would run any other module.
+
+.. code-block:: txt
+
+    torchtrtc [input_file_path] [output_file_path]
+      [input_specs...] {OPTIONS}
+
+      torchtrtc is a compiler for TorchScript, it will compile and optimize
+      TorchScript programs to run on NVIDIA GPUs using TensorRT
+
+    OPTIONS:
+
+        -h, --help                        Display this help menu
+        Verbiosity of the compiler
+          -v, --verbose                     Dumps debugging information about the
+                                            compilation process onto the console
+          -w, --warnings                    Disables warnings generated during
+                                            compilation onto the console (warnings
+                                            are on by default)
+          --i, --info                       Dumps info messages generated during
+                                            compilation onto the console
+        --build-debuggable-engine         Creates a debuggable engine
+        --allow-gpu-fallback              (Only used when targeting DLA
+                                          (device-type)) Lets engine run layers on
+                                          GPU if they are not supported on DLA
+        --require-full-compilation        Require that the model should be fully
+                                          compiled to TensorRT or throw an error
+        --check-method-support=[method_name]
+                                          Check the support for end to end
+                                          compilation of a specified method in the
+                                          TorchScript module
+        --disable-tf32                    Prevent Float32 layers from using the
+                                          TF32 data format
+        --sparse-weights                  Enable sparsity for weights of conv and
+                                          FC layers
+        -p[precision...],
+        --enable-precision=[precision...] (Repeatable) Enabling an operating
+                                          precision for kernels to use when
+                                          building the engine (Int8 requires a
+                                          calibration-cache argument) [ float |
+                                          float32 | f32 | fp32 | half | float16 |
+                                          f16 | fp16 | int8 | i8 | char ]
+                                          (default: float)
+        -d[type], --device-type=[type]    The type of device the engine should be
+                                          built for [ gpu | dla ] (default: gpu)
+        --gpu-id=[gpu_id]                 GPU id if running on multi-GPU platform
+                                          (defaults to 0)
+        --dla-core=[dla_core]             DLACore id if running on available DLA
+                                          (defaults to 0)
+        --engine-capability=[capability]  The type of device the engine should be
+                                          built for [ standard | safety |
+                                          dla_standalone ]
+        --calibration-cache-file=[file_path]
+                                          Path to calibration cache file to use
+                                          for post training quantization
+        --teo=[op_name...],
+        --torch-executed-op=[op_name...]  (Repeatable) Operator in the graph that
+                                          should always be run in PyTorch for
+                                          execution (partial compilation must be
+                                          enabled)
+        --tem=[module_name...],
+        --torch-executed-mod=[module_name...]
+                                          (Repeatable) Module that should always
+                                          be run in Pytorch for execution (partial
+                                          compilation must be enabled)
+        --mbs=[num_ops],
+        --min-block-size=[num_ops]        Minimum number of contiguous TensorRT
+                                          supported ops to compile a subgraph to
+                                          TensorRT
+        --embed-engine                    Whether to treat input file as a
+                                          serialized TensorRT engine and embed it
+                                          into a TorchScript module (device spec
+                                          must be provided)
+        --num-avg-timing-iters=[num_iters]
+                                          Number of averaging timing iterations
+                                          used to select kernels
+        --workspace-size=[workspace_size] Maximum size of workspace given to
+                                          TensorRT
+        --dla-sram-size=[dla_sram_size]   Fast software managed RAM used by DLA
+                                          to communicate within a layer.
+        --dla-local-dram-size=[dla_local_dram_size]  Host RAM used by DLA to share
+                                          intermediate tensor data across operations.
+        --dla-global-dram-size=[dla_global_dram_size] Host RAM used by DLA to store
+                                          weights and metadata for execution
+        --atol=[atol]                     Absolute tolerance threshold for acceptable
+                                          numerical deviation from standard torchscript
+                                          output (default 1e-8)
+        --rtol=[rtol]                     Relative tolerance threshold for acceptable
+                                          numerical deviation from standard torchscript
+                                          output  (default 1e-5)
+        --no-threshold-check              Skip checking threshold compliance
+        --truncate-long-double,
+        --truncate, --truncate-64bit      Truncate weights that are provided in
+                                          64bit to 32bit (Long, Double to Int,
+                                          Float)
+        --save-engine                     Instead of compiling a full a
+                                          TorchScript program, save the created
+                                          engine to the path specified as the
+                                          output path
+        --custom-torch-ops                (repeatable) Shared object/DLL containing custom torch operators
+        --custom-converters               (repeatable) Shared object/DLL containing custom converters
+        input_file_path                   Path to input TorchScript file
+        output_file_path                  Path for compiled TorchScript (or
+                                          TensorRT engine) file
+        input_specs...                    Specs for inputs to engine, can either
+                                          be a single size or a range defined by
+                                          Min, Optimal, Max sizes, e.g.
+                                          "(N,..,C,H,W)"
+                                          "[(MIN_N,..,MIN_C,MIN_H,MIN_W);(OPT_N,..,OPT_C,OPT_H,OPT_W);(MAX_N,..,MAX_C,MAX_H,MAX_W)]".
+                                          Data Type and format can be specified by
+                                          adding an "@" followed by dtype and "%"
+                                          followed by format to the end of the
+                                          shape spec. e.g. "(3, 3, 32,
+                                          32)@f16%NHWC"
+        "--" can be used to terminate flag options and force all following
+        arguments to be treated as positional options
+
+e.g.
+
+.. code-block:: shell
+
+    torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts "[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@f16%contiguous" -p f16
+
+
+To run with custom torch operators
+.. code-block:: shell
+torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts --custom-torch-ops= "[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@fp16%contiguous" -p f16
+
+
+To run with custom converters
+.. code-block:: shell
+torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts --custom-converters= "[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@fp16%contiguous" -p f16
diff --git a/docs/_sources/contributors/partitioning.rst.txt b/docs/_sources/contributors/partitioning.rst.txt
index 9f3fbacd1c..2d027c57f4 100644
--- a/docs/_sources/contributors/partitioning.rst.txt
+++ b/docs/_sources/contributors/partitioning.rst.txt
@@ -90,4 +90,4 @@ To enable automatic fallback feature, you can set following attributes in Python
       cfg.min_block_size = 2;
       cfg.torch_executed_ops.push_back("aten::relu");
       auto trt_mod = torchtrt::ts::compile(mod, cfg);
-      auto out = trt_mod.forward({in});
\ No newline at end of file
+      auto out = trt_mod.forward({in});
diff --git a/docs/_sources/getting_started/getting_started_with_cpp_api.rst.txt b/docs/_sources/getting_started/getting_started_with_cpp_api.rst.txt
new file mode 100644
index 0000000000..7f7f60a669
--- /dev/null
+++ b/docs/_sources/getting_started/getting_started_with_cpp_api.rst.txt
@@ -0,0 +1,338 @@
+.. _getting_started_cpp:
+
+Using Torch-TensorRT in  C++
+==============================
+
+If you haven't already, acquire a tarball of the library by following the instructions in :ref:`Installation`
+
+Using Torch-TensorRT in C++
+***************************
+Torch-TensorRT C++ API accepts TorchScript modules (generated either from ``torch.jit.script`` or ``torch.jit.trace``) as an input and returns
+a Torchscript module (optimized using TensorRT). This requires users to use Pytorch (in python) to generate torchscript modules beforehand.
+Please refer to `Creating TorchScript modules in Python `_ section to generate torchscript graphs.
+
+
+.. _torch_tensorrt_quickstart:
+
+[Torch-TensorRT Quickstart] Compiling TorchScript Modules with ``torchtrtc``
+---------------------------------------------------------------------------------
+
+An easy way to get started with Torch-TensorRT and to check if your model can be supported without extra work is to run it through
+``torchtrtc``, which supports almost all features of the compiler from the command line including post training quantization
+(given a previously created calibration cache). For example we can compile our lenet model by setting our preferred operating
+precision and input size. This new TorchScript file can be loaded into Python (note: you need to ``import torch_tensorrt`` before loading
+these compiled modules because the compiler extends the PyTorch the deserializer and runtime to execute compiled modules).
+
+.. code-block:: shell
+
+    ❯ torchtrtc -p f16 lenet_scripted.ts trt_lenet_scripted.ts "(1,1,32,32)"
+
+    ❯ python3
+    Python 3.6.9 (default, Apr 18 2020, 01:56:04)
+    [GCC 8.4.0] on linux
+    Type "help", "copyright", "credits" or "license" for more information.
+    >>> import torch
+    >>> import torch_tensorrt
+    >>> ts_model = torch.jit.load(“trt_lenet_scripted.ts”)
+    >>> ts_model(torch.randn((1,1,32,32)).to(“cuda”).half())
+
+You can learn more about ``torchtrtc`` usage here: :ref:`torchtrtc`
+
+.. _ts_in_cc:
+
+Working with TorchScript in C++
+--------------------------------
+
+If we are developing an application to deploy with C++, we can save either our traced or scripted module using ``torch.jit.save``
+which will serialize the TorchScript code, weights and other information into a package. This is also where our dependency on Python ends.
+
+.. code-block:: python
+
+    torch_script_module.save("lenet.jit.pt")
+
+From here we can now load our TorchScript module in C++
+
+.. code-block:: c++
+
+    #include  // One-stop header.
+
+    #include 
+    #include 
+
+    int main(int argc, const char* argv[]) {
+        torch::jit::Module module;
+        try {
+            // Deserialize the ScriptModule from a file using torch::jit::load().
+            module = torch::jit::load("");
+        }
+        catch (const c10::Error& e) {
+            std::cerr << "error loading the model\n";
+            return -1;
+        }
+
+        std::cout << "ok\n";
+
+
+You can do full training and inference in C++ with PyTorch / LibTorch if you would like, you can even define your modules in C++ and
+have access to the same powerful tensor library that backs PyTorch. (For more information: https://pytorch.org/cppdocs/).
+For instance we can do inference with our LeNet module like this:
+
+.. code-block:: c++
+
+        mod.eval();
+        torch::Tensor in = torch::randn({1, 1, 32, 32});
+        auto out = mod.forward(in);
+
+and to run on the GPU:
+
+.. code-block:: c++
+
+        mod.eval();
+        mod.to(torch::kCUDA);
+        torch::Tensor in = torch::randn({1, 1, 32, 32}, torch::kCUDA);
+        auto out = mod.forward(in);
+
+As you can see it is pretty similar to the Python API. When you call the ``forward`` method, you invoke the PyTorch JIT compiler, which will optimize and run your TorchScript code.
+
+.. _compile_cpp:
+
+Compiling with Torch-TensorRT in C++
+-------------------------------------
+We are also at the point were we can compile and optimize our module with Torch-TensorRT, but instead of in a JIT fashion we must do it ahead-of-time (AOT) i.e. before we start doing actual inference work
+since it takes a bit of time to optimize the module, it would not make sense to do this every time you run the module or even the first time you run it.
+
+With our module loaded, we can feed it into the Torch-TensorRT compiler. When we do so we must provide some information on the expected input size and also configure any additional settings.
+
+.. code-block:: c++
+
+    #include "torch/script.h"
+    #include "torch_tensorrt/torch_tensorrt.h"
+    ...
+
+        mod.to(at::kCUDA);
+        mod.eval();
+
+        auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA});
+        auto trt_mod = torch_tensorrt::CompileGraph(mod, std::vector{{in.sizes()}});
+        auto out = trt_mod.forward({in});
+
+Thats it! Now the graph runs primarily not with the JIT compiler but using TensorRT (though we execute the graph using the JIT runtime).
+
+We can also set settings like operating precision to run in FP16.
+
+.. code-block:: c++
+
+    #include "torch/script.h"
+    #include "torch_tensorrt/torch_tensorrt.h"
+    ...
+
+        mod.to(at::kCUDA);
+        mod.eval();
+
+        auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA}).to(torch::kHALF);
+        auto input_sizes = std::vector({in.sizes()});
+        torch_tensorrt::CompileSpec info(input_sizes);
+        info.enable_precisions.insert(torch::kHALF);
+        auto trt_mod = torch_tensorrt::CompileGraph(mod, info);
+        auto out = trt_mod.forward({in});
+
+And now we are running the module in FP16 precision. You can then save the module to load later.
+
+.. code-block:: c++
+
+    trt_mod.save("")
+
+Torch-TensorRT compiled TorchScript modules are loaded in the same way as normal TorchScript module. Make sure your deployment application is linked against ``libtorchtrt.so``
+
+.. code-block:: c++
+
+    #include "torch/script.h"
+    #include "torch_tensorrt/torch_tensorrt.h"
+
+    int main(int argc, const char* argv[]) {
+        torch::jit::Module module;
+        try {
+            // Deserialize the ScriptModule from a file using torch::jit::load().
+            module = torch::jit::load("");
+        }
+        catch (const c10::Error& e) {
+            std::cerr << "error loading the model\n";
+            return -1;
+        }
+
+        torch::Tensor in = torch::randn({1, 1, 32, 32}, torch::kCUDA);
+        auto out = mod.forward(in);
+
+        std::cout << "ok\n";
+    }
+
+If you want to save the engine produced by Torch-TensorRT to use in a TensorRT application you can use the ``ConvertGraphToTRTEngine`` API.
+
+.. code-block:: c++
+
+    #include "torch/script.h"
+    #include "torch_tensorrt/torch_tensorrt.h"
+    ...
+
+        mod.to(at::kCUDA);
+        mod.eval();
+
+        auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA}).to(torch::kHALF);
+        auto input_sizes = std::vector({in.sizes()});
+        torch_tensorrt::CompileSpec info(input_sizes);
+        info.enabled_precisions.insert(torch::kHALF);
+        auto trt_mod = torch_tensorrt::ConvertGraphToTRTEngine(mod, "forward", info);
+        std::ofstream out("/tmp/engine_converted_from_jit.trt");
+        out << engine;
+        out.close();
+
+.. _under_the_hood:
+
+Under The Hood
+---------------
+
+When a module is provided to Torch-TensorRT, the compiler starts by mapping a graph like you saw above to a graph like this:
+
+.. code-block:: none
+
+    graph(%input.2 : Tensor):
+        %2 : Float(84, 10) = prim::Constant[value=]()
+        %3 : Float(120, 84) = prim::Constant[value=]()
+        %4 : Float(576, 120) = prim::Constant[value=]()
+        %5 : int = prim::Constant[value=-1]() # x.py:25:0
+        %6 : int[] = prim::Constant[value=annotate(List[int], [])]()
+        %7 : int[] = prim::Constant[value=[2, 2]]()
+        %8 : int[] = prim::Constant[value=[0, 0]]()
+        %9 : int[] = prim::Constant[value=[1, 1]]()
+        %10 : bool = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+        %11 : int = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+        %12 : bool = prim::Constant[value=0]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+        %self.classifer.fc3.bias : Float(10) = prim::Constant[value= 0.0464  0.0383  0.0678  0.0932  0.1045 -0.0805 -0.0435 -0.0818  0.0208 -0.0358 [ CUDAFloatType{10} ]]()
+        %self.classifer.fc2.bias : Float(84) = prim::Constant[value=]()
+        %self.classifer.fc1.bias : Float(120) = prim::Constant[value=]()
+        %self.feat.conv2.weight : Float(16, 6, 3, 3) = prim::Constant[value=]()
+        %self.feat.conv2.bias : Float(16) = prim::Constant[value=]()
+        %self.feat.conv1.weight : Float(6, 1, 3, 3) = prim::Constant[value=]()
+        %self.feat.conv1.bias : Float(6) = prim::Constant[value= 0.0530 -0.1691  0.2802  0.1502  0.1056 -0.1549 [ CUDAFloatType{6} ]]()
+        %input0.4 : Tensor = aten::_convolution(%input.2, %self.feat.conv1.weight, %self.feat.conv1.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+        %input0.5 : Tensor = aten::relu(%input0.4) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+        %input1.2 : Tensor = aten::max_pool2d(%input0.5, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+        %input0.6 : Tensor = aten::_convolution(%input1.2, %self.feat.conv2.weight, %self.feat.conv2.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+        %input2.1 : Tensor = aten::relu(%input0.6) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+        %x.1 : Tensor = aten::max_pool2d(%input2.1, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+        %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0
+        %27 : Tensor = aten::matmul(%input.1, %4)
+        %28 : Tensor = trt::const(%self.classifer.fc1.bias)
+        %29 : Tensor = aten::add_(%28, %27, %11)
+        %input0.2 : Tensor = aten::relu(%29) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+        %31 : Tensor = aten::matmul(%input0.2, %3)
+        %32 : Tensor = trt::const(%self.classifer.fc2.bias)
+        %33 : Tensor = aten::add_(%32, %31, %11)
+        %input1.1 : Tensor = aten::relu(%33) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+        %35 : Tensor = aten::matmul(%input1.1, %2)
+        %36 : Tensor = trt::const(%self.classifer.fc3.bias)
+        %37 : Tensor = aten::add_(%36, %35, %11)
+        return (%37)
+    (CompileGraph)
+
+The graph has now been transformed from a collection of modules, each managing their own parameters into a single graph with the parameters inlined
+into the graph and all of the operations laid out. Torch-TensorRT has also executed a number of optimizations and mappings to make the graph easier to translate to TensorRT.
+From here the compiler can assemble the TensorRT engine by following the dataflow through the graph.
+
+When the graph construction phase is complete, Torch-TensorRT produces a serialized TensorRT engine. From here depending on the API, this engine is returned
+to the user or moves into the graph construction phase. Here Torch-TensorRT creates a JIT Module to execute the TensorRT engine which will be instantiated and managed
+by the Torch-TensorRT runtime.
+
+Here is the graph that you get back after compilation is complete:
+
+.. code-block:: none
+
+    graph(%self_1 : __torch__.lenet, %input_0 : Tensor):
+        %1 : ...trt.Engine = prim::GetAttr[name="lenet"](%self_1)
+        %3 : Tensor[] = prim::ListConstruct(%input_0)
+        %4 : Tensor[] = trt::execute_engine(%3, %1)
+        %5 : Tensor = prim::ListUnpack(%4)
+        return (%5)
+
+
+You can see the call where the engine is executed, after extracting the attribute containing the engine and constructing a list of inputs, then returns the tensors back to the user.
+
+.. _unsupported_ops:
+
+Working with Unsupported Operators
+-----------------------------------
+
+Torch-TensorRT is a new library and the PyTorch operator library is quite large, so there will be ops that aren't supported natively by the compiler. You can either use the composition techinques
+shown above to make modules are fully Torch-TensorRT supported and ones that are not and stitch the modules together in the deployment application or you can register converters for missing ops.
+
+    You can check support without going through the full compilation pipleine using the ``torch_tensorrt::CheckMethodOperatorSupport(const torch::jit::Module& module, std::string method_name)`` api
+    to see what operators are not supported. ``torchtrtc`` automatically checks modules with this method before starting compilation and will print out a list of operators that are not supported.
+
+.. _custom_converters:
+
+Registering Custom Converters
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Operations are mapped to TensorRT through the use of modular converters, a function that takes a node from a the JIT graph and produces an equivalent layer or subgraph in TensorRT.
+Torch-TensorRT ships with a library of these converters stored in a registry, that will be executed depending on the node being parsed. For instance a ``aten::relu(%input0.4)`` instruction will trigger
+the relu converter to be run on it, producing an activation layer in the TensorRT graph. But since this library is not exhaustive you may need to write your own to get Torch-TensorRT
+to support your module.
+
+Shipped with the Torch-TensorRT distribution are the internal core API headers. You can therefore access the converter registry and add a converter for the op you need.
+
+For example, if we try to compile a graph with a build of Torch-TensorRT that doesn't support the flatten operation (``aten::flatten``) you may see this error:
+
+.. code-block:: none
+
+    terminate called after throwing an instance of 'torch_tensorrt::Error'
+    what():  [enforce fail at core/conversion/conversion.cpp:109] Expected converter to be true but got false
+    Unable to convert node: %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0 (conversion.AddLayer)
+    Schema: aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor)
+    Converter for aten::flatten requested, but no such converter was found.
+    If you need a converter for this operator, you can try implementing one yourself
+    or request a converter: https://www.github.com/NVIDIA/Torch-TensorRT/issues
+
+We can register a converter for this operator in our application. All of the tools required to build a converter can be imported by including ``torch_tensorrt/core/conversion/converters/converters.h``.
+We start by creating an instance of the self-registering class ``torch_tensorrt::core::conversion::converters::RegisterNodeConversionPatterns()`` which will register converters
+in the global converter registry, associating a function schema like ``aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor)`` with a lambda that
+will take the state of the conversion, the node/operation in question to convert and all of the inputs to the node and produces as a side effect a new layer in the TensorRT network.
+Arguments are passed as a vector of inspectable unions of TensorRT ``ITensors`` and Torch ``IValues`` in the order arguments are listed in the schema.
+
+Below is a implementation of a ``aten::flatten`` converter that we can use in our application. You have full access to the Torch and TensorRT libraries in the converter implementation. So
+for example we can quickly get the output size by just running the operation in PyTorch instead of implementing the full calculation outself like we do below for this flatten converter.
+
+.. code-block:: c++
+
+    #include "torch/script.h"
+    #include "torch_tensorrt/torch_tensorrt.h"
+    #include "torch_tensorrt/core/conversion/converters/converters.h"
+
+    static auto flatten_converter = torch_tensorrt::core::conversion::converters::RegisterNodeConversionPatterns()
+        .pattern({
+            "aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor)",
+            [](torch_tensorrt::core::conversion::ConversionCtx* ctx,
+               const torch::jit::Node* n,
+               torch_tensorrt::core::conversion::converters::args& args) -> bool {
+                auto in = args[0].ITensor();
+                auto start_dim = args[1].unwrapToInt();
+                auto end_dim = args[2].unwrapToInt();
+                auto in_shape = torch_tensorrt::core::util::toVec(in->getDimensions());
+                auto out_shape = torch::flatten(torch::rand(in_shape), start_dim, end_dim).sizes();
+
+                auto shuffle = ctx->net->addShuffle(*in);
+                shuffle->setReshapeDimensions(torch_tensorrt::core::util::toDims(out_shape));
+                shuffle->setName(torch_tensorrt::core::util::node_info(n).c_str());
+
+                auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], shuffle->getOutput(0));
+                return true;
+            }
+        });
+
+    int main() {
+        ...
+
+To use this converter in Python, it is recommended to use PyTorch's `C++ / CUDA Extention `_
+template to wrap your library of converters into a ``.so`` that you can load with ``ctypes.CDLL()`` in your Python application.
+
+You can find more information on all the details of writing converters in the contributors documentation (:ref:`writing_converters`).
+If you find yourself with a large library of converter implementations, do consider upstreaming them, PRs are welcome and it would be great for the community to benefit as well.
diff --git a/docs/_sources/getting_started/getting_started_with_python_api.rst.txt b/docs/_sources/getting_started/getting_started_with_python_api.rst.txt
new file mode 100644
index 0000000000..fece176156
--- /dev/null
+++ b/docs/_sources/getting_started/getting_started_with_python_api.rst.txt
@@ -0,0 +1,58 @@
+.. _getting_started_with_python_api:
+
+Using Torch-TensorRT in Python
+*******************************
+
+The Torch-TensorRT Python API supports a number of unique usecases compared to the CLI and C++ APIs which solely support TorchScript compilation.
+
+Torch-TensorRT Python API can accept a ``torch.nn.Module``, ``torch.jit.ScriptModule``, or ``torch.fx.GraphModule`` as an input.
+Depending on what is provided one of the two frontends (TorchScript or FX) will be selected to compile the module. Provided the
+module type is supported, users may explicitly set which frontend they would like to use using the ``ir`` flag for ``compile``.
+If given a ``torch.nn.Module`` and the ``ir`` flag is set to either ``default`` or ``torchscript`` the module will be run through
+``torch.jit.script`` to convert the input module into a  TorchScript module.
+
+
+To compile your input ``torch.nn.Module`` with Torch-TensorRT, all you need to do is provide the module and inputs
+to Torch-TensorRT and you will be returned an optimized TorchScript module to run or add into another PyTorch module. Inputs
+is a list of ``torch_tensorrt.Input`` classes which define input's shape, datatype and memory format. You can also specify settings such as
+operating precision for the engine or target device. After compilation you can save the module just like any other module
+to load in a deployment application. In order to load a TensorRT/TorchScript module, make sure you first import ``torch_tensorrt``.
+
+.. code-block:: python
+
+    import torch_tensorrt
+
+    ...
+
+    model = MyModel().eval()  # torch module needs to be in eval (not training) mode
+
+    inputs = [
+        torch_tensorrt.Input(
+            min_shape=[1, 1, 16, 16],
+            opt_shape=[1, 1, 32, 32],
+            max_shape=[1, 1, 64, 64],
+            dtype=torch.half,
+        )
+    ]
+    enabled_precisions = {torch.float, torch.half}  # Run with fp16
+
+    trt_ts_module = torch_tensorrt.compile(
+        model, inputs=inputs, enabled_precisions=enabled_precisions
+    )
+
+    input_data = input_data.to("cuda").half()
+    result = trt_ts_module(input_data)
+    torch.jit.save(trt_ts_module, "trt_ts_module.ts")
+
+.. code-block:: python
+
+    # Deployment application
+    import torch
+    import torch_tensorrt
+
+    trt_ts_module = torch.jit.load("trt_ts_module.ts")
+    input_data = input_data.to("cuda").half()
+    result = trt_ts_module(input_data)
+
+Torch-TensorRT Python API also provides ``torch_tensorrt.ts.compile`` which accepts a TorchScript module as input and ``torch_tensorrt.fx.compile`` which accepts a FX GraphModule as input.
+
diff --git a/docs/_sources/getting_started/installation.rst.txt b/docs/_sources/getting_started/installation.rst.txt
new file mode 100644
index 0000000000..937dd1dde9
--- /dev/null
+++ b/docs/_sources/getting_started/installation.rst.txt
@@ -0,0 +1,362 @@
+.. _installation:
+
+Installation
+=============
+
+Precompiled Binaries
+*********************
+
+Dependencies
+---------------
+
+You need to have either PyTorch or LibTorch installed based on if you are using Python or C++
+and you must have CUDA, cuDNN and TensorRT installed.
+
+    * https://www.pytorch.org
+    * https://developer.nvidia.com/cuda
+    * https://developer.nvidia.com/cudnn
+    * https://developer.nvidia.com/tensorrt
+
+
+Python Package
+---------------
+
+You can install the python package using
+
+.. code-block:: sh
+
+    pip3 install torch-tensorrt -f https://github.com/pytorch/TensorRT/releases
+
+.. _bin-dist:
+
+C++ Binary Distribution
+------------------------
+
+Precompiled tarballs for releases are provided here: https://github.com/pytorch/TensorRT/releases
+
+.. _compile-from-source:
+
+Compiling From Source
+******************************************
+
+.. _installing-deps:
+
+Dependencies for Compilation
+-------------------------------
+
+Torch-TensorRT is built with Bazel, so begin by installing it.
+
+    * The easiest way is to install bazelisk using the method of your choosing https://github.com/bazelbuild/bazelisk
+    * Otherwise you can use the following instructions to install binaries https://docs.bazel.build/versions/master/install.html
+    * Finally if you need to compile from source (e.g. aarch64 until bazel distributes binaries for the architecture) you can use these instructions
+
+    .. code-block:: shell
+
+        export BAZEL_VERSION=$(cat /.bazelversion)
+        mkdir bazel
+        cd bazel
+        curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip
+        unzip bazel-$BAZEL_VERSION-dist.zip
+        bash ./compile.sh
+        cp output/bazel /usr/local/bin/
+
+
+You will also need to have CUDA installed on the system (or if running in a container, the system must have
+the CUDA driver installed and the container must have CUDA)
+
+The correct LibTorch version will be pulled down for you by bazel.
+
+    NOTE: For best compatability with official PyTorch, use torch==1.10.0+cuda113, TensorRT 8.0 and cuDNN 8.2 for CUDA 11.3 however Torch-TensorRT itself supports
+    TensorRT and cuDNN for other CUDA versions for usecases such as using NVIDIA compiled distributions of PyTorch that use other versions of CUDA
+    e.g. aarch64 or custom compiled version of PyTorch.
+
+.. _abis:
+
+Choosing the Right ABI
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Likely the most complicated thing about compiling Torch-TensorRT is selecting the correct ABI. There are two options
+which are incompatible with each other, pre-cxx11-abi and the cxx11-abi. The complexity comes from the fact that while
+the most popular distribution of PyTorch (wheels downloaded from pytorch.org/pypi directly) use the pre-cxx11-abi, most
+other distributions you might encounter (e.g. ones from NVIDIA - NGC containers, and builds for Jetson as well as certain
+libtorch builds and likely if you build PyTorch from source) use the cxx11-abi. It is important you compile Torch-TensorRT
+using the correct ABI to function properly. Below is a table with general pairings of PyTorch distribution sources and the
+recommended commands:
+
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| PyTorch Source                                              | Recommended C++ Compilation Command                      | Recommended Python Compilation Command                             |
++=============================================================+==========================================================+====================================================================+
+| PyTorch whl file from PyTorch.org                           | bazel build //:libtorchtrt -c opt --config pre_cxx11_abi | python3 setup.py bdist_wheel                                       |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| libtorch-shared-with-deps-*.zip from PyTorch.org            | bazel build //:libtorchtrt -c opt --config pre_cxx11_abi | python3 setup.py bdist_wheel                                       |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| libtorch-cxx11-abi-shared-with-deps-*.zip from PyTorch.org  | bazel build //:libtorchtrt -c opt                        | python3 setup.py bdist_wheel --use-cxx11-abi                       |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| PyTorch preinstalled in an NGC container                    | bazel build //:libtorchtrt -c opt                        | python3 setup.py bdist_wheel --use-cxx11-abi                       |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| PyTorch from the NVIDIA Forums for Jetson                   | bazel build //:libtorchtrt -c opt                        | python3 setup.py bdist_wheel --jetpack-version 4.6 --use-cxx11-abi |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+| PyTorch built from Source                                   | bazel build //:libtorchtrt -c opt                        | python3 setup.py bdist_wheel --use-cxx11-abi                       |
++-------------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------------------+
+
+    NOTE: For all of the above cases you must correctly declare the source of PyTorch you intend to use in your WORKSPACE file for both Python and C++ builds. See below for more information
+
+You then have two compilation options:
+
+.. _build-from-archive:
+
+**Building using cuDNN & TensorRT tarball distributions**
+--------------------------------------------------------------
+
+    This is recommended so as to build Torch-TensorRT hermetically and insures any compilation errors are not caused by version issues
+
+    Make sure when running Torch-TensorRT that these versions of the libraries are prioritized in your ``$LD_LIBRARY_PATH``
+
+You need to download the tarball distributions of TensorRT and cuDNN from the NVIDIA website.
+    * https://developer.nvidia.com/cudnn
+    * https://developer.nvidia.com/tensorrt
+
+Place these files in a directory (the directories ``third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]`` exist for this purpose)
+
+Then compile referencing the directory with the tarballs
+
+    If you get errors regarding the packages, check their sha256 hashes and make sure they match the ones listed in ``WORKSPACE``
+
+Release Build
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt -c opt --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
+A tarball with the include files and library can then be found in ``bazel-bin``
+
+.. _build-from-archive-debug:
+
+Debug Build
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To build with debug symbols use the following command
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt -c dbg --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
+A tarball with the include files and library can then be found in ``bazel-bin``
+
+Pre CXX11 ABI Build
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To build using the pre-CXX11 ABI use the ``pre_cxx11_abi`` config
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt --config pre_cxx11_abi -c [dbg/opt] --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
+A tarball with the include files and library can then be found in ``bazel-bin``
+
+.. _build-from-local:
+
+**Building using locally installed cuDNN & TensorRT**
+--------------------------------------------------------------
+
+    If you encounter bugs and you compiled using this method please disclose that you used local sources in the issue (an ldd dump would be nice too)
+
+Install TensorRT, CUDA and cuDNN on the system before starting to compile.
+
+In WORKSPACE comment out:
+
+.. code-block:: python
+
+    # Downloaded distributions to use with --distdir
+    http_archive(
+        name="cudnn",
+        urls=[
+            "",
+        ],
+        build_file="@//third_party/cudnn/archive:BUILD",
+        sha256="",
+        strip_prefix="cuda",
+    )
+
+    http_archive(
+        name="tensorrt",
+        urls=[
+            "",
+        ],
+        build_file="@//third_party/tensorrt/archive:BUILD",
+        sha256="",
+        strip_prefix="TensorRT-",
+    )
+
+and uncomment
+
+.. code-block:: python
+
+    # Locally installed dependencies
+    new_local_repository(
+        name="cudnn", path="/usr/", build_file="@//third_party/cudnn/local:BUILD"
+    )
+
+    new_local_repository(
+        name="tensorrt", path="/usr/", build_file="@//third_party/tensorrt/local:BUILD"
+    )
+
+Release Build
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Compile using:
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt -c opt
+
+A tarball with the include files and library can then be found in ``bazel-bin``
+
+.. _build-from-local-debug:
+
+Debug Build
+^^^^^^^^^^^^
+
+To build with debug symbols use the following command
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt -c dbg
+
+
+A tarball with the include files and library can then be found in ``bazel-bin``
+
+Pre CXX11 ABI Build
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To build using the pre-CXX11 ABI use the ``pre_cxx11_abi`` config
+
+.. code-block:: shell
+
+    bazel build //:libtorchtrt --config pre_cxx11_abi -c [dbg/opt]
+
+**Building with CMake**
+-----------------------
+
+It is possible to build the API libraries (in cpp/) and the torchtrtc executable using CMake instead of Bazel.
+Currently, the python API and the tests cannot be built with CMake.
+Begin by installing CMake.
+
+    * Latest releases of CMake and instructions on how to install are available for different platforms
+      [on their website](https://cmake.org/download/).
+
+A few useful CMake options include:
+
+    * CMake finders for TensorRT and cuDNN are provided in `cmake/Modules`. In order for CMake to use them, pass
+      `-DCMAKE_MODULE_PATH=cmake/Modules` when configuring the project with CMake.
+    * Libtorch provides its own CMake finder. In case CMake doesn't find it, pass the path to your install of
+      libtorch with `-DTorch_DIR=/share/cmake/Torch`
+    * If TensorRT is not found with the provided cmake finder, specify `-DTensorRT_ROOT=`
+    * Finally, configure and build the project in a build directory of your choice with the following command
+      from the root of Torch-TensorRT project:
+
+    .. code-block:: shell
+
+        cmake -S. -B \
+            [-DCMAKE_MODULE_PATH=cmake/Module] \
+            [-DTorch_DIR=/share/cmake/Torch] \
+            [-DTensorRT_ROOT=] \
+            [-DCMAKE_BUILD_TYPE=Debug|Release]
+        cmake --build 
+
+**Building the Python package**
+--------------------------------
+
+Begin by installing ``ninja``
+
+You can build the Python package using ``setup.py`` (this will also build the correct version of ``libtorchtrt.so``)
+
+.. code-block:: shell
+
+    python3 setup.py [install/bdist_wheel]
+
+Debug Build
+^^^^^^^^^^^^
+
+.. code-block:: shell
+
+    python3 setup.py develop [--user]
+
+This also compiles a debug build of ``libtorchtrt.so``
+
+**Building Natively on aarch64 (Jetson)**
+-------------------------------------------
+
+Prerequisites
+^^^^^^^^^^^^^^
+
+Install or compile a build of PyTorch/LibTorch for aarch64
+
+NVIDIA hosts builds the latest release branch for Jetson here:
+
+    https://forums.developer.nvidia.com/t/pytorch-for-jetson-version-1-10-now-available/72048
+
+
+Enviorment Setup
+^^^^^^^^^^^^^^^^^
+
+To build natively on aarch64-linux-gnu platform, configure the ``WORKSPACE`` with local available dependencies.
+
+1. Replace ``WORKSPACE`` with the corresponding WORKSPACE file in ``//toolchains/jp_workspaces``
+
+2. Configure the correct paths to directory roots containing local dependencies in the ``new_local_repository`` rules:
+
+    NOTE: If you installed PyTorch using a pip package, the correct path is the path to the root of the python torch package.
+    In the case that you installed with ``sudo pip install`` this will be ``/usr/local/lib/python3.8/dist-packages/torch``.
+    In the case you installed with ``pip install --user`` this will be ``$HOME/.local/lib/python3.8/site-packages/torch``.
+
+In the case you are using NVIDIA compiled pip packages, set the path for both libtorch sources to the same path. This is because unlike
+PyTorch on x86_64, NVIDIA aarch64 PyTorch uses the CXX11-ABI. If you compiled for source using the pre_cxx11_abi and only would like to
+use that library, set the paths to the same path but when you compile make sure to add the flag ``--config=pre_cxx11_abi``
+
+.. code-block:: shell
+
+    new_local_repository(
+        name = "libtorch",
+        path = "/usr/local/lib/python3.8/dist-packages/torch",
+        build_file = "third_party/libtorch/BUILD"
+    )
+
+    new_local_repository(
+        name = "libtorch_pre_cxx11_abi",
+        path = "/usr/local/lib/python3.8/dist-packages/torch",
+        build_file = "third_party/libtorch/BUILD"
+    )
+
+
+Compile C++ Library and Compiler CLI
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    NOTE: Due to shifting dependency locations between Jetpack 4.5 and 4.6 there is a now a flag to inform bazel of the Jetpack version
+
+    .. code-block:: shell
+
+        --platforms //toolchains:jetpack_x.x
+
+
+Compile Torch-TensorRT library using bazel command:
+
+.. code-block:: shell
+
+   bazel build //:libtorchtrt --platforms //toolchains:jetpack_5.0
+
+Compile Python API
+^^^^^^^^^^^^^^^^^^^^
+
+    NOTE: Due to shifting dependencies locations between Jetpack 4.5 and newer Jetpack verisons there is now a flag for ``setup.py`` which sets the jetpack version (default: 5.0)
+
+Compile the Python API using the following command from the ``//py`` directory:
+
+.. code-block:: shell
+
+    python3 setup.py install --use-cxx11-abi
+
+If you have a build of PyTorch that uses Pre-CXX11 ABI drop the ``--use-cxx11-abi`` flag
+
+If you are building for Jetpack 4.5 add the ``--jetpack-version 5.0`` flag
diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt
index 9e9c87d23b..c0de8dd400 100644
--- a/docs/_sources/index.rst.txt
+++ b/docs/_sources/index.rst.txt
@@ -22,44 +22,43 @@ More Information / System Architecture:
 Getting Started
 ----------------
 * :ref:`installation`
-* :ref:`getting_started`
+* :ref:`getting_started_with_python_api`
+* :ref:`getting_started_cpp`
+
+.. toctree::
+   :caption: Getting Started
+   :maxdepth: 1
+   :hidden:
+
+   getting_started/installation
+   getting_started/getting_started_with_python_api
+   getting_started/getting_started_with_cpp_api
+
+
+Tutorials
+------------
+* :ref:`creating_a_ts_mod`
+* :ref:`getting_started_with_fx`
 * :ref:`ptq`
-* :ref:`torchtrtc`
-* :ref:`use_from_pytorch`
 * :ref:`runtime`
-* :ref:`using_dla`
 * :ref:`serving_torch_tensorrt_with_triton`
+* :ref:`use_from_pytorch`
+* :ref:`using_dla`
+* :ref:`notebooks`
 
 .. toctree::
-   :caption: Getting Started
+   :caption: Tutorials
    :maxdepth: 1
    :hidden:
 
-   tutorials/installation
-   tutorials/getting_started_with_cpp_api
-   tutorials/getting_started_with_python_api
    tutorials/creating_torchscript_module_in_python
+   tutorials/getting_started_with_fx_path
    tutorials/ptq
-   tutorials/torchtrtc
-   tutorials/use_from_pytorch
    tutorials/runtime
-   tutorials/using_dla
    tutorials/serving_torch_tensorrt_with_triton
-
-.. toctree::
-   :caption: Notebooks
-   :maxdepth: 1
-   :hidden:
-
-   _notebooks/CitriNet-example
-   _notebooks/dynamic-shapes
-   _notebooks/EfficientNet-example
-   _notebooks/Hugging-Face-BERT
-   _notebooks/lenet-getting-started
-   _notebooks/Resnet50-example
-   _notebooks/ssd-object-detection-demo
-   _notebooks/vgg-qat
-
+   tutorials/use_from_pytorch
+   tutorials/using_dla
+   tutorials/notebooks
 
 Python API Documenation
 ------------------------
@@ -67,6 +66,7 @@ Python API Documenation
 * :ref:`torch_tensorrt_logging_py`
 * :ref:`torch_tensorrt_ptq_py`
 * :ref:`torch_tensorrt_ts_py`
+* :ref:`torch_tensorrt_fx_py`
 
 .. toctree::
    :caption: Python API Documenation
@@ -77,6 +77,7 @@ Python API Documenation
    py_api/logging
    py_api/ptq
    py_api/ts
+   py_api/fx
 
 C++ API Documenation
 ----------------------
@@ -97,6 +98,18 @@ C++ API Documenation
    _cpp_api/namespace_torch_tensorrt__torchscript
    _cpp_api/namespace_torch_tensorrt__ptq
 
+CLI Documentation
+---------------------
+* :ref:`torchtrtc`
+
+.. toctree::
+   :caption: CLI Documenation
+   :maxdepth: 0
+   :hidden:
+
+   cli/torchtrtc
+
+
 Contributor Documentation
 --------------------------------
 * :ref:`system_overview`
diff --git a/docs/_sources/py_api/fx.rst.txt b/docs/_sources/py_api/fx.rst.txt
new file mode 100644
index 0000000000..8ce591f5ee
--- /dev/null
+++ b/docs/_sources/py_api/fx.rst.txt
@@ -0,0 +1,31 @@
+.. _torch_tensorrt_fx_py:
+
+torch_tensorrt.fx
+===================
+
+.. currentmodule:: torch_tensorrt.fx
+
+.. automodule torch_tensorrt.ts
+   :undoc-members:
+
+.. automodule:: torch_tensorrt.fx
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Functions
+------------
+
+.. autofunction:: compile
+
+
+Classes
+--------
+
+.. autoclass:: TRTModule
+
+.. autoclass:: InputTensorSpec
+
+.. autoclass:: TRTInterpreter
+
+.. autoclass:: TRTInterpreterResult
\ No newline at end of file
diff --git a/docs/_sources/py_api/ptq.rst.txt b/docs/_sources/py_api/ptq.rst.txt
index 7686209cd9..ec83662efb 100644
--- a/docs/_sources/py_api/ptq.rst.txt
+++ b/docs/_sources/py_api/ptq.rst.txt
@@ -24,4 +24,4 @@ Classes
 Enums
 -------
 
-.. autoclass:: CalibrationAlgo
\ No newline at end of file
+.. autoclass:: CalibrationAlgo
diff --git a/docs/_sources/py_api/torch_tensorrt.rst.txt b/docs/_sources/py_api/torch_tensorrt.rst.txt
index 199621a663..2ccf7c60e2 100644
--- a/docs/_sources/py_api/torch_tensorrt.rst.txt
+++ b/docs/_sources/py_api/torch_tensorrt.rst.txt
@@ -57,4 +57,4 @@ Submodules
    logging
    ptq
    ts
-
+   fx
diff --git a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/demo.rst.txt b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/demo.rst.txt
index 66f19cc899..3ad6e46b15 100644
--- a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/demo.rst.txt
+++ b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/demo.rst.txt
@@ -78,7 +78,7 @@ Meta
    :description lang=en: A demonstration of the reStructuredText
        markup language, containing examples of all basic
        constructs and many advanced constructs.
-
+       
 Blocks
 ======
 
@@ -282,7 +282,7 @@ Glossary
 This is a glossary with definition terms for thing like :term:`Writing`:
 
 .. glossary::
-
+  
   Documentation
      Provides users with the knowledge they need to use something.
 
diff --git a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/lists_tables.rst.txt b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/lists_tables.rst.txt
index a31d213924..4c7642e5c5 100644
--- a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/lists_tables.rst.txt
+++ b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/lists_tables.rst.txt
@@ -150,7 +150,7 @@ Second list level
   - here is an inner bullet ``oh``
 
     - one more ``with an inline literally``. `yahoo `_
-
+      
       heh heh. child. try to beat this embed:
 
       .. literalinclude:: test_py_module/test.py
diff --git a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/structure.rst.txt b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/structure.rst.txt
index d21fc96321..178ed63f07 100644
--- a/docs/_sources/src/pytorch-sphinx-theme/docs/demo/structure.rst.txt
+++ b/docs/_sources/src/pytorch-sphinx-theme/docs/demo/structure.rst.txt
@@ -17,7 +17,7 @@ Aliquam accumsan feugiat ligula, nec fringilla libero commodo sed. Proin et erat
 Etiam turpis ante, luctus sed velit tristique, finibus volutpat dui. Nam sagittis vel ante nec malesuada.
 Praesent dignissim mi nec ornare elementum. Nunc eu augue vel sem dignissim cursus sed et nulla.
 Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.
-Pellentesque dictum dui sem, non placerat tortor rhoncus in. Sed placerat nulla at rhoncus iaculis.
+Pellentesque dictum dui sem, non placerat tortor rhoncus in. Sed placerat nulla at rhoncus iaculis. 
 
 Document Section
 ================
@@ -49,7 +49,7 @@ pretium tristique lectus. Sed pellentesque leo lectus, et convallis ipsum euismo
 Integer at leo vitae felis pretium aliquam fringilla quis odio. Sed pharetra enim accumsan feugiat pretium.
 Maecenas at pharetra tortor. Morbi semper eget mi vel finibus. Cras rutrum nulla eros, id feugiat arcu pellentesque ut.
 Sed finibus tortor ac nisi ultrices viverra. Duis feugiat malesuada sapien, at commodo ante porttitor ac.
-Curabitur posuere mauris mi, vel ornare orci scelerisque sit amet. Suspendisse nec fringilla dui.
+Curabitur posuere mauris mi, vel ornare orci scelerisque sit amet. Suspendisse nec fringilla dui. 
 
 Document Paragraph
 """"""""""""""""""
@@ -66,7 +66,7 @@ Structural Elements 2
 Etiam turpis ante, luctus sed velit tristique, finibus volutpat dui. Nam sagittis vel ante nec malesuada.
 Praesent dignissim mi nec ornare elementum. Nunc eu augue vel sem dignissim cursus sed et nulla.
 Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.
-Pellentesque dictum dui sem, non placerat tortor rhoncus in. Sed placerat nulla at rhoncus iaculis.
+Pellentesque dictum dui sem, non placerat tortor rhoncus in. Sed placerat nulla at rhoncus iaculis. 
 
 Document Section
 ================
@@ -98,4 +98,4 @@ Etiam efficitur sollicitudin libero, tincidunt volutpat ligula interdum sed. Pra
 Vivamus sagittis risus et egestas commodo.Cras venenatis arcu in pharetra interdum.
 Donec quis metus porttitor tellus cursus lobortis. Quisque et orci magna. Fusce rhoncus mi mi,
 at vehicula massa rhoncus quis. Mauris augue leo, pretium eget molestie vitae, efficitur nec nulla.
-In hac habitasse platea dictumst. Sed sit amet imperdiet purus.
+In hac habitasse platea dictumst. Sed sit amet imperdiet purus. 
diff --git a/docs/_sources/tutorials/creating_torchscript_module_in_python.rst.txt b/docs/_sources/tutorials/creating_torchscript_module_in_python.rst.txt
index eaa1708b83..1d1cdba574 100644
--- a/docs/_sources/tutorials/creating_torchscript_module_in_python.rst.txt
+++ b/docs/_sources/tutorials/creating_torchscript_module_in_python.rst.txt
@@ -17,6 +17,7 @@ For example, we can define a LeNet module like this:
     import torch.nn as nn
     import torch.nn.functional as F
 
+
     class LeNetFeatExtractor(nn.Module):
         def __init__(self):
             super(LeNetFeatExtractor, self).__init__()
@@ -28,6 +29,7 @@ For example, we can define a LeNet module like this:
             x = F.max_pool2d(F.relu(self.conv2(x)), 2)
             return x
 
+
     class LeNetClassifier(nn.Module):
         def __init__(self):
             super(LeNetClassifier, self).__init__()
@@ -36,12 +38,13 @@ For example, we can define a LeNet module like this:
             self.fc3 = nn.Linear(84, 10)
 
         def forward(self, x):
-            x = torch.flatten(x,1)
+            x = torch.flatten(x, 1)
             x = F.relu(self.fc1(x))
             x = F.relu(self.fc2(x))
             x = self.fc3(x)
             return x
 
+
     class LeNet(nn.Module):
         def __init__(self):
             super(LeNet, self).__init__()
@@ -67,7 +70,7 @@ To trace an instance of our LeNet module, we can call ``torch.jit.trace`` with a
     import torch
 
     model = LeNet()
-    input_data = torch.empty([1,1,32,32])
+    input_data = torch.empty([1, 1, 32, 32])
     traced_model = torch.jit.trace(model, input_data)
 
 Scripting actually inspects your code with a compiler and generates an equivalent TorchScript program. The difference is that since tracing
diff --git a/docs/_sources/tutorials/getting_started_with_fx_path.rst.txt b/docs/_sources/tutorials/getting_started_with_fx_path.rst.txt
new file mode 100644
index 0000000000..d7dc9b0166
--- /dev/null
+++ b/docs/_sources/tutorials/getting_started_with_fx_path.rst.txt
@@ -0,0 +1,297 @@
+.. _getting_started_with_fx:
+
+Torch-TensorRT (FX Frontend) User Guide
+========================
+Torch-TensorRT (FX Frontend) is a tool that can convert a PyTorch model through ``torch.fx`` to an
+TensorRT engine optimized targeting running on Nvidia GPUs. TensorRT is the inference engine
+developed by NVIDIA which composed of various kinds of optimization including kernel fusion,
+graph optimization, low precision, etc.. This tool is developed in Python environment which allows this
+workflow to be very accessible to researchers and engineers. There are a few stages that a
+user want to use this tool and we will introduce them here.
+
+> Torch-TensorRT (FX Frontend) is in ``Beta`` and currently it is recommended to work with PyTorch nightly.
+
+.. code-block:: shell
+
+    # Test an example by
+    $ python py/torch_tensorrt/fx/example/lower_example.py
+
+
+Converting a PyTorch Model to TensorRT Engine
+---------------------------------------------
+In general, users are welcome to use the ``compile()`` to finish the conversion from a model to tensorRT engine. It is a
+wrapper API that consists of the major steps needed to finish this converison. Please refer to ``lower_example.py`` file in ``examples/fx``.
+
+In this section, we will go through an example to illustrate the major steps that fx path uses.
+Users can refer to ``fx2trt_example.py`` file in ``examples/fx``.
+
+* **Step 1: Trace the model with acc_tracer**
+Acc_tracer is a tracer inheritated from FX tracer. It comes with args normalizer to convert all args to kwargs and pass to TRT converters.
+
+.. code-block:: shell
+
+   import torch_tensorrt.fx.tracer.acc_tracer.acc_tracer as acc_tracer
+
+   # Build the model which needs to be a PyTorch nn.Module.
+   my_pytorch_model = build_model()
+
+   # Prepare inputs to the model. Inputs have to be a List of Tensors
+   inputs = [Tensor, Tensor, ...]
+
+   # Trace the model with acc_tracer.
+   acc_mod = acc_tracer.trace(my_pytorch_model, inputs)
+
+*Common Errors:*
+
+symbolically traced variables cannot be used as inputs to control flow
+This means the model contains dynamic control flow. Please refer to section “Dynamic Control Flow” in `FX guide `_.
+
+* **Step 2: Build TensorRT engine**
+There are `two different modes `_ for how TensorRT handles batch dimension, explicit batch dimension and implicit batch dimension. This mode was used by early versions of TensorRT, and is now deprecated but continues to be supported for backwards compatibility. In explicit batch mode, all dimensions are explicit and can be dynamic, that is their length can change at execution time. Many new features, such as dynamic shapes and loops, are available only in this mode. User can still choose to use implicit batch mode when they set ``explicit_batch_dimension=False`` in ``compile()``. We do not recommend to use it since it will lack of support in future TensorRT versions.
+
+Explicit batch is the default mode and it must be set for dynamic shape. For most of vision task, user can choose to enable ``dynamic_batch`` in ``compile()`` if they want to get the similar effects as implicit mode where only batch dimension changes. It has some requirements:
+1. Shapes of inputs, outputs and activations are fixed except batch dimension.
+2. Inputs, outputs and activations have batch dimension as the major dimension.
+3. All the operators in the model do not modify batch dimension (permute, transpose, split, etc.) or compute over batch dimension (sum, softmax, etc.).
+
+For examples of the last path, if we have a 3D tensor t shaped as (batch, sequence, dimension), operations such as torch.transpose(0, 2). If any of these three are not satisfied, we’ll need to specify InputTensorSpec as inputs with dynamic range.
+
+.. code-block:: shell
+
+    import deeplearning.trt.fx2trt.converter.converters
+    from torch.fx.experimental.fx2trt.fx2trt import InputTensorSpec, TRTInterpreter
+
+    # InputTensorSpec is a dataclass we use to store input information.
+    # There're two ways we can build input_specs.
+    # Option 1, build it manually.
+    input_specs = [
+      InputTensorSpec(shape=(1, 2, 3), dtype=torch.float32),
+      InputTensorSpec(shape=(1, 4, 5), dtype=torch.float32),
+    ]
+    # Option 2, build it using sample_inputs where user provide a sample
+    inputs = [
+    torch.rand((1,2,3), dtype=torch.float32),
+    torch.rand((1,4,5), dtype=torch.float32),
+    ]
+    input_specs = InputTensorSpec.from_tensors(inputs)
+
+    # IMPORTANT: If dynamic shape is needed, we need to build it slightly differently.
+    input_specs = [
+        InputTensorSpec(
+            shape=(-1, 2, 3),
+            dtype=torch.float32,
+            # Currently we only support one set of dynamic range. User may set other dimensions but it is not promised to work for any models
+            # (min_shape, optimize_target_shape, max_shape)
+            # For more information refer to fx/input_tensor_spec.py
+            shape_ranges = [
+                ((1, 2, 3), (4, 2, 3), (100, 2, 3)),
+            ],
+        ),
+        InputTensorSpec(shape=(1, 4, 5), dtype=torch.float32),
+    ]
+
+    # Build a TRT interpreter. Set explicit_batch_dimension accordingly.
+    interpreter = TRTInterpreter(
+        acc_mod, input_specs, explicit_batch_dimension=True/False
+    )
+
+    # The output of TRTInterpreter run() is wrapped as TRTInterpreterResult.
+    # The TRTInterpreterResult contains required parameter to build TRTModule,
+    # and other informational output from TRTInterpreter run.
+    class TRTInterpreterResult(NamedTuple):
+        engine: Any
+        input_names: Sequence[str]
+        output_names: Sequence[str]
+        serialized_cache: bytearray
+
+    #max_batch_size: set accordingly for maximum batch size you will use.
+    #max_workspace_size: set to the maximum size we can afford for temporary buffer
+    #lower_precision: the precision model layers are running on (TensorRT will choose the best perforamnce precision).
+    #sparse_weights: allow the builder to examine weights and use optimized functions when weights have suitable sparsity
+    #force_fp32_output: force output to be fp32
+    #strict_type_constraints: Usually we should set it to False unless we want to control the precision of certain layer for numeric #reasons.
+    #algorithm_selector: set up algorithm selection for certain layer
+    #timing_cache: enable timing cache for TensorRT
+    #profiling_verbosity: TensorRT logging level
+    trt_interpreter_result = interpreter.run(
+        max_batch_size=64,
+        max_workspace_size=1 << 25,
+        sparse_weights=False,
+        force_fp32_output=False,
+        strict_type_constraints=False,
+        algorithm_selector=None,
+        timing_cache=None,
+        profiling_verbosity=None,
+    )
+
+
+*Common Errors:*
+
+RuntimeError: Conversion of function xxx not currently supported!
+- This means we don’t have the support for this xxx operator. Please refer to section “How to add a missing op” below for further instructions.
+
+* **Step 3: Run the model**
+One way is using TRTModule, which is basically a PyTorch nn.Module.
+
+.. code-block:: shell
+
+    from torch_tensorrt.fx import TRTModule
+    mod = TRTModule(
+        trt_interpreter_result.engine,
+        trt_interpreter_result.input_names,
+        trt_interpreter_result.output_names)
+    # Just like all other PyTorch modules
+    outputs = mod(*inputs)
+    torch.save(mod, "trt.pt")
+    reload_trt_mod = torch.load("trt.pt")
+    reload_model_output = reload_trt_mod(*inputs)
+
+So far, we give a detailed explanation of major steps in convterting a PyTorch model into TensorRT engine. Users are welcome to refer to the source code for some parameters explanations. In the converting scheme, there are two important actions in it. One is acc tracer which helps us to convert a PyTorch model to acc graph. The other is FX path converter which helps to convert the acc graph's operation to corresponding TensorRT operation and build up the TensoRT engine for it.
+
+Acc Tracer
+---------
+
+Acc tracer is a custom FX symbolic tracer. It does a couple more things compare to the vanilla FX symbolic tracer. We mainly depend on it to convert PyTorch ops or builtin ops to acc ops. There are two main purposes for fx2trt to use acc ops:
+
+1. there’re many ops that do similar things in PyTorch ops and builtin ops such like torch.add, builtin.add and torch.Tensor.add. Using acc tracer, we normalize these three ops to a single acc_ops.add. This helps reduce the number of converters we need to write.
+2. acc ops only have kwargs which makes writing converter easier as we don’t need to add additional logic to find arguments in args and kwargs.
+
+FX2TRT
+--------
+After symbolic tracing, we have the graph representation of a PyTorch model. fx2trt leverages the power of fx.Interpreter. fx.Interpreter goes through the whole graph node by node and calls the function that node represents. fx2trt overrides the original behavior of calling the function with invoking corresponding converts for each node. Each converter function adds corresponding TensorRT layer(s).
+
+Below is an example of a converter function. The decorator is used to register this converter function with the corresponding node. In this example, we register this converter to a fx node whose target is acc_ops.sigmoid.
+
+.. code-block:: shell
+
+    @tensorrt_converter(acc_ops.sigmoid)
+    def acc_ops_sigmoid(network, target, args, kwargs, name):
+        """
+        network: TensorRT network. We'll be adding layers to it.
+
+        The rest arguments are attributes of fx node.
+        """
+        input_val = kwargs['input']
+
+        if not isinstance(input_val, trt.tensorrt.ITensor):
+            raise RuntimeError(f'Sigmoid received input {input_val} that is not part '
+                            'of the TensorRT region!')
+
+        layer = network.add_activation(input=input_val, type=trt.ActivationType.SIGMOID)
+        layer.name = name
+        return layer.get_output(0)
+
+How to Add a Missing Op
+****************
+
+You can actually add it wherever you want just need to remember import the file so that all acc ops and mapper will be registered before tracing with acc_tracer.
+
+* **Step 1. Add a new acc op**
+
+TODO: Need to explain more on the logistic of acc op like when we want to break down an op and when we want to reuse other ops.
+
+In `acc tracer `_, we convert nodes in the graph to acc ops if there’s a mapping registered for the node to an acc op.
+
+In order to make the conversion to acc ops to happen, there’re two things required. One is that there should be an acc op function defined and the other is there should be a mapping registered.
+
+Defining an acc op is simple, we first just need a function and register the function as an acc op via this decorator `acc_normalizer.py `_. e.g. the following code adds an acc op named foo() which adds two given inputs.
+
+.. code-block:: shell
+
+    # NOTE: all acc ops should only take kwargs as inputs, therefore we need the "*"
+    # at the beginning.
+    @register_acc_op
+    def foo(*, input, other, alpha):
+        return input + alpha * other
+
+There’re two ways to register a mapping. One is `register_acc_op_mapping() `_. Let’s register a mapping from torch.add to foo() we just created above. We need to add decorator register_acc_op_mapping to it.
+
+.. code-block:: shell
+
+    this_arg_is_optional = True
+
+    @register_acc_op_mapping(
+        op_and_target=("call_function", torch.add),
+        arg_replacement_tuples=[
+            ("input", "input"),
+            ("other", "other"),
+            ("alpha", "alpha", this_arg_is_optional),
+        ],
+    )
+    @register_acc_op
+    def foo(*, input, other, alpha=1.0):
+        return input + alpha * other
+
+``op_and_target`` determines which node will trigger this mapping. op and target are the attributes of FX node. In acc_normalization when we see a node with the same op and target as set in the ``op_and_target``, we will trigger the mapping. Since we want to map from ``torch.add``, then op would be call_function and target would be ``torch.add``. ``arg_replacement_tuples`` determines how we construct kwargs for new acc op node using args and kwargs from original node. Each tuple in ``arg_replacement_tuples`` represents one argument mapping rule. It contains two or three elements. The third element is a boolean variable that determines whether this kwarg is optional in *original node*. We only need to specify the third element if it’s True. The first element is the argument name in original node which will be used as the acc op node’s argument whose name is the second element in the tuple. The sequence of the tuples does matter because the position of the tuple determines where the argument is in original node’s args. We use this information to map args from original node to kwargs in acc op node.
+We don’t have to specify arg_replacement_tuples if none of the followings are true.
+
+1. kwargs of original nodes and acc op nodes have different name.
+2. there’re optional arguments.
+
+The other way to register a mapping is through `register_custom_acc_mapper_fn() `_. This one is designed to reduce the redundant op registration as it allows you to use a function to map to one or more existing acc ops throught some combinations. In the function, you can do basically whatever you want. Let’s use an example to explain how it works.
+
+.. code-block:: shell
+
+    @register_acc_op
+    def foo(*, input, other, alpha=1.0):
+        return input + alpha * other
+
+    @register_custom_acc_mapper_fn(
+        op_and_target=("call_function", torch.add),
+        arg_replacement_tuples=[
+            ("input", "input"),
+            ("other", "other"),
+            ("alpha", "alpha", this_arg_is_optional),
+        ],
+    )
+    def custom_mapper(node: torch.fx.Node, _: nn.Module) -> torch.fx.Node:
+        """
+        `node` is original node, which is a call_function node with target
+        being torch.add.
+        """
+        alpha = 1
+        if "alpha" in node.kwargs:
+            alpha = node.kwargs["alpha"]
+        foo_kwargs = {"input": node["input"], "other": node["other"], "alpha": alpha}
+        with node.graph.inserting_before(node):
+            foo_node = node.graph.call_function(foo, kwargs=foo_kwargs)
+            foo_node.meta = node.meta.copy()
+            return foo_node
+
+
+In the custom mapper function, we construct an acc op node and return it. The node we returns here would take over all the children nodes of original nodes `acc_normalizer.py `_.
+
+The last step would be *adding unit test* for the new acc op or mapper function we added. The place to add the unit test is here `test_acc_tracer.py `_.
+
+* **Step 2. Add a new converter**
+
+All the developed converters for acc ops are all in `acc_op_converter.py `_. It could give you a good example of how the converter is added.
+
+Essentially, the converter is the mapping mechanism that maps the acc ops to a TensorRT layer. If we are able to find all the TensorRT layers we need we can get start to add a converter for the node using `TensorRT APIs `_.
+
+.. code-block:: shell
+
+    @tensorrt_converter(acc_ops.sigmoid)
+    def acc_ops_sigmoid(network, target, args, kwargs, name):
+        """
+        network: TensorRT network. We'll be adding layers to it.
+
+        The rest arguments are attributes of fx node.
+        """
+        input_val = kwargs['input']
+
+        if not isinstance(input_val, trt.tensorrt.ITensor):
+            raise RuntimeError(f'Sigmoid received input {input_val} that is not part '
+                            'of the TensorRT region!')
+
+        layer = network.add_activation(input=input_val, type=trt.ActivationType.SIGMOID)
+        layer.name = name
+        return layer.get_output(0)
+
+We need to use ``tensorrt_converter`` decorator to register the converter. The argument for the decorator is the target of the fx node that we need to convert. In the converter, we can find the inputs to the fx node in kwargs. As in the example, the original node is `acc_ops.sigmoid` which only has one argument “input” in acc_ops.py. We get the input and check if it’s a TensorRT tensor. After that, we add a sigmoid layer to TensorRT network and return the output of the layer. The output we returned will be passed to the children nodes of acc_ops.sigmoid by fx.Interpreter.
+
+**What if we can not find corresponding layers in TensorRT that do the same thing as the node.**
+
+In this case, we would need to do a bit more work. TensorRT provides plugins which serves as custom layers. *We have not implement this feature yet. We will update once it is enabled*.
+
+Last step would be adding the unit test for the new converter we added. User could add corresponding unit test in this `folder `_.
diff --git a/docs/_sources/tutorials/notebooks.rst.txt b/docs/_sources/tutorials/notebooks.rst.txt
new file mode 100644
index 0000000000..df903fc353
--- /dev/null
+++ b/docs/_sources/tutorials/notebooks.rst.txt
@@ -0,0 +1,154 @@
+.. _notebooks:
+
+Example notebooks
+===================
+
+There exists a number of notebooks which cover specific using specific features and models
+with Torch-TensorRT
+
+Notebooks
+------------
+
+Compiling CitriNet with Torch-TensorRT
+********************************************
+
+Citrinet is an acoustic model used for the speech to text recognition task. It is a version
+of QuartzNet that extends ContextNet, utilizing subword encoding (via Word Piece tokenization)
+and Squeeze-and-Excitation(SE) mechanism and are therefore smaller than QuartzNet models. CitriNet
+models take in audio segments and transcribe them to letter, byte pair, or word piece sequences.
+
+This notebook demonstrates the steps for optimizing a pretrained CitriNet model with Torch-TensorRT,
+and running it to test the speedup obtained.
+
+* `Torch-TensorRT Getting Started - CitriNet `_
+
+
+Compiling EfficentNet with Torch-TensorRT
+********************************************
+
+EfficentNet is a feedforward CNN designed to achieve better performance and accuracy than alternative architectures
+by using a "scaling method that uniformly scales all dimensions of depth/width/resolution using a simple yet highly effective compound coefficient".
+
+This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT,
+and running it to test the speedup obtained.
+
+* `Torch-TensorRT Getting Started - EfficientNet-B0 `_
+
+
+Masked Language Modeling (MLM) with Hugging Face BERT Transformer accelerated by Torch-TensorRT
+*************************************************************************************************
+
+"BERT is a transformer model pretrained on a large corpus of English data in a self-supervised fashion.
+This way, the model learns an inner representation of the English language that can then be used to extract
+features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train
+a standard classifier using the features produced by the BERT model as inputs." (https://huggingface.co/bert-base-uncased)
+
+This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT,
+and running it to test the speedup obtained.
+
+* `Masked Language Modeling (MLM) with Hugging Face BERT Transformer `_
+
+
+Serving a model in C++ using Torch-TensorRT
+**********************************************
+
+This example shows how you can load a pretrained ResNet-50 model, convert it to a Torch-TensorRT
+optimized model (via the Torch-TensorRT Python API), save the model as a torchscript module, and
+then finally load and serve the model with the PyTorch C++ API.
+
+* `ResNet C++ Serving Example `_
+
+
+Compiling ResNet50 with Torch-TensorRT
+********************************************
+
+This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a
+pretrained ResNet-50 network, and running it to test the speedup obtained.
+
+* `Torch-TensorRT Getting Started - ResNet 50 `_
+
+
+Using Dynamic Shapes with Torch-TensorRT
+********************************************
+
+Making use of Dynamic Shaped Tensors in Torch TensorRT is quite simple. Let's say you are
+using the ``torch_tensorrt.compile(...)`` function  to compile a torchscript module. One
+of the args in this function in this function is ``input``: which defines an input to a
+module in terms of expected shape, data type and tensor format: ``torch_tensorrt.Input.``
+
+For the purposes of this walkthrough we just need three kwargs: `min_shape`, `opt_shape`` and `max_shape`.
+
+.. code-block:: py
+
+    torch_tensorrt.Input(
+            min_shape=(1, 224, 224, 3),
+            opt_shape=(1, 512, 512, 3),
+            max_shape=(1, 1024, 1024, 3),
+            dtype=torch.int32
+            format=torch.channel_last
+        )
+    ...
+
+In this example, we are going to use a simple ResNet model to demonstrate the use of the API.
+
+* `Torch-TensorRT - Using Dynamic Shapes `_
+
+Using the FX Frontend with Torch-TensorRT
+********************************************
+
+The purpose of this example is to demostrate the overall flow of lowering a PyTorch model to TensorRT
+conveniently with using FX.
+
+* `Using the FX Frontend with Torch-TensorRT `_
+
+
+Compiling a PyTorch model using FX Frontend with Torch-TensorRT
+*******************************************************************
+
+The purpose of this example is to demonstrate the overall flow of lowering a PyTorch
+model to TensorRT via FX with existing FX based tooling
+
+* `Compiling a PyTorch model using FX Frontend with Torch-TensorRT  `_
+
+
+Compiling LeNet with Torch-TensorRT
+*******************************************************************
+
+This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a simple LeNet network.
+
+* `Torch-TensorRT Getting Started - LeNet  `_
+
+
+Accelerate Deep Learning Models using Quantization in Torch-TensorRT
+*******************************************************************
+
+Model Quantization is a popular way of optimization which reduces the size of models thereby
+accelerating inference, also opening up the possibilities of deployments on devices with lower
+computation power such as Jetson. Simply put, quantization is a process of mapping input values
+ from a larger set to output values in a smaller set. In this notebook, we illustrate the workflow
+ that you can adopt while quantizing a deep learning model in Torch-TensorRT. The notebook takes
+ you through an example of Mobilenetv2 for a classification task on a subset of Imagenet Dataset
+ called Imagenette which has 10 classes.
+
+* `Accelerate Deep Learning Models using Quantization in Torch-TensorRT `_
+
+
+Object Detection with Torch-TensorRT (SSD)
+*******************************************************************
+
+This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a pretrained SSD network, and running it to test the speedup obtained.
+
+* `Object Detection with Torch-TensorRT (SSD)  `_
+
+
+Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT
+*****************************************************************************
+
+Quantization Aware training (QAT) simulates quantization during training by
+quantizing weights and activation layers. This will help to reduce the loss in
+accuracy when we convert the network trained in FP32 to INT8 for faster inference.
+QAT introduces additional nodes in the graph which will be used to learn the dynamic
+ranges of weights and activation layers. In this notebook, we illustrate the following
+steps from training to inference of a QAT model in Torch-TensorRT.
+
+* `Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT  `_
diff --git a/docs/_sources/tutorials/ptq.rst.txt b/docs/_sources/tutorials/ptq.rst.txt
index 047fc9f40f..b62457109f 100644
--- a/docs/_sources/tutorials/ptq.rst.txt
+++ b/docs/_sources/tutorials/ptq.rst.txt
@@ -130,8 +130,6 @@ Then all thats required to setup the module for INT8 calibration is to set the f
     compile_spec.enabled_precisions.insert(torch::kI8);
     /// Use the TensorRT Entropy Calibrator
     compile_spec.ptq_calibrator = calibrator;
-    /// Set a larger workspace (you may get better performace from doing so)
-    compile_spec.workspace_size = 1 << 28;
 
     auto trt_mod = torch_tensorrt::CompileGraph(mod, compile_spec);
 
@@ -150,38 +148,39 @@ a TensorRT calibrator by providing desired configuration. The following code dem
 
 .. code-block:: python
 
-    testing_dataset = torchvision.datasets.CIFAR10(root='./data',
-                                                            train=False,
-                                                            download=True,
-                                                            transform=transforms.Compose([
-                                                                transforms.ToTensor(),
-                                                                transforms.Normalize((0.4914, 0.4822, 0.4465),
-                                                                                     (0.2023, 0.1994, 0.2010))
-                                                            ]))
-
-    testing_dataloader = torch.utils.data.DataLoader(testing_dataset,
-                                                          batch_size=1,
-                                                          shuffle=False,
-                                                          num_workers=1)
-    calibrator = torch_tensorrt.ptq.DataLoaderCalibrator(testing_dataloader,
-                                                  cache_file='./calibration.cache',
-                                                  use_cache=False,
-                                                  algo_type=torch_tensorrt.ptq.CalibrationAlgo.ENTROPY_CALIBRATION_2,
-                                                  device=torch.device('cuda:0'))
-
-    compile_spec = {
-             "inputs": [torch_tensorrt.Input((1, 3, 32, 32))],
-             "enabled_precisions": {torch.float, torch.half, torch.int8},
-             "calibrator": calibrator,
-             "device": {
-                 "device_type": torch_tensorrt.DeviceType.GPU,
-                 "gpu_id": 0,
-                 "dla_core": 0,
-                 "allow_gpu_fallback": False,
-                 "disable_tf32": False
-             }
-         }
-    trt_mod = torch_tensorrt.compile(model, compile_spec)
+    testing_dataset = torchvision.datasets.CIFAR10(
+        root="./data",
+        train=False,
+        download=True,
+        transform=transforms.Compose(
+            [
+                transforms.ToTensor(),
+                transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
+            ]
+        ),
+    )
+
+    testing_dataloader = torch.utils.data.DataLoader(
+        testing_dataset, batch_size=1, shuffle=False, num_workers=1
+    )
+    calibrator = torch_tensorrt.ptq.DataLoaderCalibrator(
+        testing_dataloader,
+        cache_file="./calibration.cache",
+        use_cache=False,
+        algo_type=torch_tensorrt.ptq.CalibrationAlgo.ENTROPY_CALIBRATION_2,
+        device=torch.device("cuda:0"),
+    )
+
+    trt_mod = torch_tensorrt.compile(model, inputs=[torch_tensorrt.Input((1, 3, 32, 32))],
+                                        enabled_precisions={torch.float, torch.half, torch.int8},
+                                        calibrator=calibrator,
+                                        device={
+                                             "device_type": torch_tensorrt.DeviceType.GPU,
+                                             "gpu_id": 0,
+                                             "dla_core": 0,
+                                             "allow_gpu_fallback": False,
+                                             "disable_tf32": False
+                                         })
 
 In the cases where there is a pre-existing calibration cache file that users want to use, ``CacheCalibrator`` can be used without any dataloaders. The following example demonstrates how
 to use ``CacheCalibrator`` to use in INT8 mode.
@@ -190,13 +189,9 @@ to use ``CacheCalibrator`` to use in INT8 mode.
 
   calibrator = torch_tensorrt.ptq.CacheCalibrator("./calibration.cache")
 
-  compile_settings = {
-        "inputs": [torch_tensorrt.Input([1, 3, 32, 32])],
-        "enabled_precisions": {torch.float, torch.half, torch.int8},
-        "calibrator": calibrator,
-    }
-
-  trt_mod = torch_tensorrt.compile(model, compile_settings)
+  trt_mod = torch_tensorrt.compile(model, inputs=[torch_tensorrt.Input([1, 3, 32, 32])],
+                                        enabled_precisions={torch.float, torch.half, torch.int8},
+                                        calibrator=calibrator)
 
 If you already have an existing calibrator class (implemented directly using TensorRT API), you can directly set the calibrator field to your class which can be very convenient.
 For a demo on how PTQ can be performed on a VGG network using Torch-TensorRT API, you can refer to https://github.com/pytorch/TensorRT/blob/master/tests/py/test_ptq_dataloader_calibrator.py
diff --git a/docs/_sources/tutorials/use_from_pytorch.rst.txt b/docs/_sources/tutorials/use_from_pytorch.rst.txt
index 0c616e9414..296e4c3c32 100644
--- a/docs/_sources/tutorials/use_from_pytorch.rst.txt
+++ b/docs/_sources/tutorials/use_from_pytorch.rst.txt
@@ -32,8 +32,8 @@ at the documentation for the Torch-TensorRT ``TensorRTCompileSpec`` API.
 .. code-block:: python
 
     spec = {
-        "forward":
-            torch_tensorrt.ts.TensorRTCompileSpec({
+        "forward": torch_tensorrt.ts.TensorRTCompileSpec(
+            {
                 "inputs": [torch_tensorrt.Input([1, 3, 300, 300])],
                 "enabled_precisions": {torch.float, torch.half},
                 "refit": False,
@@ -42,13 +42,13 @@ at the documentation for the Torch-TensorRT ``TensorRTCompileSpec`` API.
                     "device_type": torch_tensorrt.DeviceType.GPU,
                     "gpu_id": 0,
                     "dla_core": 0,
-                    "allow_gpu_fallback": True
+                    "allow_gpu_fallback": True,
                 },
                 "capability": torch_tensorrt.EngineCapability.default,
-                "num_min_timing_iters": 2,
                 "num_avg_timing_iters": 1,
-            })
-        }
+            }
+        )
+    }
 
 Now to compile with Torch-TensorRT, provide the target module objects and the spec dictionary to ``torch._C._jit_to_backend("tensorrt", ...)``
 
diff --git a/docs/_sources/tutorials/using_dla.rst.txt b/docs/_sources/tutorials/using_dla.rst.txt
index ab40b942b3..9b062f7053 100644
--- a/docs/_sources/tutorials/using_dla.rst.txt
+++ b/docs/_sources/tutorials/using_dla.rst.txt
@@ -33,9 +33,6 @@ Using DLA in a C++ application
     # If a layer fails to run on DLA it will fallback to GPU
     compile_spec.device.allow_gpu_fallback = true;
 
-    # Set the workspace size
-    compile_spec.workspace_size = 1 << 28;
-
 
 Using DLA in a python application
 
@@ -44,7 +41,7 @@ Using DLA in a python application
     compile_spec = {
         "inputs": [torch_tensorrt.Input(self.input.shape)],
         "device": torch_tensorrt.Device("dla:0", allow_gpu_fallback=True),
-        "enalbed_precisions": {torch.half}
+        "enalbed_precisions": {torch.half},
     }
 
     trt_mod = torch_tensorrt.compile(self.scripted_model, compile_spec)
diff --git a/docs/_static/css/theme.css b/docs/_static/css/theme.css
index 42b5713933..e4a55c2ac9 100644
--- a/docs/_static/css/theme.css
+++ b/docs/_static/css/theme.css
@@ -10715,6 +10715,32 @@ article.pytorch-article table {
   table-layout: fixed;
 }
 
+div.deprecated p {
+  display: inline;
+}
+div.deprecated,
+div.versionchanged {
+  margin-top: 0.5rem;
+  padding: 0.5rem;
+  margin-bottom: 0.5rem;
+  border: none;
+}
+div.versionadded {
+  margin: 1rem 0;
+}
+div.deprecated p:last-child,
+div.versionchanged p:last-child,
+div.versionadded p:last-child {
+  margin-bottom: 0
+}
+div.deprecated {
+  color: #b94a48;
+  background-color: #fdede9;
+}
+div.versionchanged {
+  background-color: #fffbe8;
+}
+
 article.pytorch-article .note,
 article.pytorch-article .warning,
 article.pytorch-article .tip,
diff --git a/docs/cli/torchtrtc.html b/docs/cli/torchtrtc.html
new file mode 100644
index 0000000000..931194b7fd
--- /dev/null
+++ b/docs/cli/torchtrtc.html
@@ -0,0 +1,776 @@
+
+
+
+
+
+  
+
+  
+  
+
+  
+  
+  torchtrtc — Torch-TensorRT master documentation
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+  
+  
+  
+    
+    
+    
+    
+  
+  
+  
+  
+
+  
+  
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+
+
+
+
+   
+
+    
+
+    
    
+      Table of Contents
+      
+    
    
+
+    
+
+    
    
+      
    
+        
    
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    
+
+  
+
+  
+
    
+        
    
+
+        
    
+          Shortcuts
+        
    
+      
    
+
+      
    
+        
    
+
+        
+          
+          
    
+          
+            
    
+             
    
+              
+  
    
+
    torchtrtc
    
+
    torchtrtc is a CLI application for using the Torch-TensorRT compiler. It serves as an easy way to compile a
+TorchScript Module with Torch-TensorRT from the command-line to quickly check support or as part of
+a deployment pipeline. All basic features of the compiler are supported including post training
+quantization (though you must already have a calibration cache file to use the PTQ feature). The compiler can
+output two formats, either a TorchScript program with the TensorRT engine embedded or
+the TensorRT engine itself as a PLAN file.
    
+
    All that is required to run the program after compilation is for C++ linking against libtorchtrt.so
+or in Python importing the torch_tensorrt package. All other aspects of using compiled modules are identical
+to standard TorchScript. Load with torch.jit.load() and run like you would run any other module.
    
+
    torchtrtc [input_file_path] [output_file_path]
+  [input_specs...] {OPTIONS}
+
+  torchtrtc is a compiler for TorchScript, it will compile and optimize
+  TorchScript programs to run on NVIDIA GPUs using TensorRT
+
+OPTIONS:
+
+    -h, --help                        Display this help menu
+    Verbiosity of the compiler
+      -v, --verbose                     Dumps debugging information about the
+                                        compilation process onto the console
+      -w, --warnings                    Disables warnings generated during
+                                        compilation onto the console (warnings
+                                        are on by default)
+      --i, --info                       Dumps info messages generated during
+                                        compilation onto the console
+    --build-debuggable-engine         Creates a debuggable engine
+    --allow-gpu-fallback              (Only used when targeting DLA
+                                      (device-type)) Lets engine run layers on
+                                      GPU if they are not supported on DLA
+    --require-full-compilation        Require that the model should be fully
+                                      compiled to TensorRT or throw an error
+    --check-method-support=[method_name]
+                                      Check the support for end to end
+                                      compilation of a specified method in the
+                                      TorchScript module
+    --disable-tf32                    Prevent Float32 layers from using the
+                                      TF32 data format
+    --sparse-weights                  Enable sparsity for weights of conv and
+                                      FC layers
+    -p[precision...],
+    --enable-precision=[precision...] (Repeatable) Enabling an operating
+                                      precision for kernels to use when
+                                      building the engine (Int8 requires a
+                                      calibration-cache argument) [ float |
+                                      float32 | f32 | fp32 | half | float16 |
+                                      f16 | fp16 | int8 | i8 | char ]
+                                      (default: float)
+    -d[type], --device-type=[type]    The type of device the engine should be
+                                      built for [ gpu | dla ] (default: gpu)
+    --gpu-id=[gpu_id]                 GPU id if running on multi-GPU platform
+                                      (defaults to 0)
+    --dla-core=[dla_core]             DLACore id if running on available DLA
+                                      (defaults to 0)
+    --engine-capability=[capability]  The type of device the engine should be
+                                      built for [ standard | safety |
+                                      dla_standalone ]
+    --calibration-cache-file=[file_path]
+                                      Path to calibration cache file to use
+                                      for post training quantization
+    --teo=[op_name...],
+    --torch-executed-op=[op_name...]  (Repeatable) Operator in the graph that
+                                      should always be run in PyTorch for
+                                      execution (partial compilation must be
+                                      enabled)
+    --tem=[module_name...],
+    --torch-executed-mod=[module_name...]
+                                      (Repeatable) Module that should always
+                                      be run in Pytorch for execution (partial
+                                      compilation must be enabled)
+    --mbs=[num_ops],
+    --min-block-size=[num_ops]        Minimum number of contiguous TensorRT
+                                      supported ops to compile a subgraph to
+                                      TensorRT
+    --embed-engine                    Whether to treat input file as a
+                                      serialized TensorRT engine and embed it
+                                      into a TorchScript module (device spec
+                                      must be provided)
+    --num-avg-timing-iters=[num_iters]
+                                      Number of averaging timing iterations
+                                      used to select kernels
+    --workspace-size=[workspace_size] Maximum size of workspace given to
+                                      TensorRT
+    --dla-sram-size=[dla_sram_size]   Fast software managed RAM used by DLA
+                                      to communicate within a layer.
+    --dla-local-dram-size=[dla_local_dram_size]  Host RAM used by DLA to share
+                                      intermediate tensor data across operations.
+    --dla-global-dram-size=[dla_global_dram_size] Host RAM used by DLA to store
+                                      weights and metadata for execution
+    --atol=[atol]                     Absolute tolerance threshold for acceptable
+                                      numerical deviation from standard torchscript
+                                      output (default 1e-8)
+    --rtol=[rtol]                     Relative tolerance threshold for acceptable
+                                      numerical deviation from standard torchscript
+                                      output  (default 1e-5)
+    --no-threshold-check              Skip checking threshold compliance
+    --truncate-long-double,
+    --truncate, --truncate-64bit      Truncate weights that are provided in
+                                      64bit to 32bit (Long, Double to Int,
+                                      Float)
+    --save-engine                     Instead of compiling a full a
+                                      TorchScript program, save the created
+                                      engine to the path specified as the
+                                      output path
+    --custom-torch-ops                (repeatable) Shared object/DLL containing custom torch operators
+    --custom-converters               (repeatable) Shared object/DLL containing custom converters
+    input_file_path                   Path to input TorchScript file
+    output_file_path                  Path for compiled TorchScript (or
+                                      TensorRT engine) file
+    input_specs...                    Specs for inputs to engine, can either
+                                      be a single size or a range defined by
+                                      Min, Optimal, Max sizes, e.g.
+                                      "(N,..,C,H,W)"
+                                      "[(MIN_N,..,MIN_C,MIN_H,MIN_W);(OPT_N,..,OPT_C,OPT_H,OPT_W);(MAX_N,..,MAX_C,MAX_H,MAX_W)]".
+                                      Data Type and format can be specified by
+                                      adding an "@" followed by dtype and "%"
+                                      followed by format to the end of the
+                                      shape spec. e.g. "(3, 3, 32,
+                                      32)@f16%NHWC"
+    "--" can be used to terminate flag options and force all following
+    arguments to be treated as positional options
+
    
+
    
+
    e.g.
    
+
    torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts "[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@f16%contiguous" -p f16
+
    
+
    
+
    To run with custom torch operators
+.. code-block:: shell
+torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts –custom-torch-ops=<path to custom library> “[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@fp16%contiguous” -p f16
    
+
    To run with custom converters
+.. code-block:: shell
+torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts –custom-converters=<path to custom library> “[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@fp16%contiguous” -p f16
    
+
    
+
+
+             
    
+             
+            
    
+            
+
+          
    
+        
    
+
+        
    
+          
    
+            
    
+              
+
+            
    
+          
    
+        
    
+      
    
+    
    
+
+  
+
+
+  
+
+     
+       
+         
+         
+         
+         
+         
+         
+         
+     
+
+  
+
+  
+  
+  
+  
+
+   
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
    Docs
    
+          
    Access comprehensive developer documentation for PyTorch
    
+          View Docs
+        
    
+
+        
    
+          
    Tutorials
    
+          
    Get in-depth tutorials for beginners and advanced developers
    
+          View Tutorials
+        
    
+
+        
    
+          
    Resources
    
+          
    Find development resources and get your questions answered
    
+          View Resources
+        
    
+      
    
+    
    
+  
    
+
+  
+
+  
    
+  
    
+    
    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.
    
+    
+  
    
+
    
+
+  
+
+  
+
+  
    
+    
    
+      
    
+        
    
+          
+          
+        
    
+      
    
+    
    
+
+    
    
+      
    
+        
+      
    
+    
    
+  
    
+
+  
+
+  
+
+  
+
+
\ No newline at end of file
diff --git a/docs/contributors/conversion.html b/docs/contributors/conversion.html
index 6e42653dc2..63cdfc41ce 100644
--- a/docs/contributors/conversion.html
+++ b/docs/contributors/conversion.html
@@ -9,22 +9,22 @@
   
 
   
-
+  
   Conversion Phase — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,32 +305,32 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -344,13 +342,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
 
    Conversion Phase
    
 
    Once the graph has be simplified to a form thats easy to convert, we then set up a conversion context
@@ -410,25 +408,25 @@ 
    Node Converters
-
+      
         
-
-
+      
+      
         
-
+      
     
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -436,11 +434,11 @@ 
    Node ConvertersSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -464,12 +462,12 @@ 
    Node Converters
          
          
@@ -478,9 +476,9 @@ 
    Node Converters
          
          
+     
 
-
-
+  
 
   
   
@@ -491,7 +489,7 @@ 
    Node Converters
 
   
-
+  
   Lowering Phase — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,32 +305,32 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -344,13 +342,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
 
    Lowering Phase
    
 
    The lowering phase is made up out of passes which are operations which map a graph from a high level representation
@@ -553,25 +551,25 @@ 
    Unroll Loops
-
+      
         
-
-
+      
+      
         
-
+      
     
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -579,11 +577,11 @@ 
    Unroll LoopsSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -626,12 +624,12 @@ 
    Unroll Loops
          
          
@@ -640,9 +638,9 @@ 
    Unroll Loops
          
          
+     
 
-
-
+  
 
   
   
@@ -653,7 +651,7 @@ 
    Unroll Loops
 
   
-
+  
   Partitioning Phase — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,32 +305,32 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -344,13 +342,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
 
    Partitioning Phase
    
 
    The phase is optional and enabled by the user. It instructs the compiler to separate nodes into ones that should run in PyTorch and ones that should run in TensorRT.
@@ -435,25 +433,25 @@ 
    Automatic Fallback
-
+      
         
-
-
+      
+      
         
-
+      
     
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -461,11 +459,11 @@ 
    Automatic FallbackSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -492,12 +490,12 @@ 
    Automatic Fallback
          
          
@@ -506,9 +504,9 @@ 
    Automatic Fallback
          
          
+     
 
-
-
+  
 
   
   
@@ -519,7 +517,7 @@ 
    Automatic Fallback
 
   
-
+  
   Compiler Phases — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -34,11 +34,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -178,9 +178,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -191,21 +191,21 @@
       
    
         
       
    
     
@@ -286,7 +284,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -305,30 +303,30 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -340,13 +338,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -446,9 +444,9 @@ 
    Compilation and Runtime
          
          
+     
 
-
-
+  
 
   
   
@@ -459,7 +457,7 @@ 
    Compilation and Runtime
+   
 
   
 
diff --git a/docs/contributors/runtime.html b/docs/contributors/runtime.html
index 0fbc5df769..f59ebcbd15 100644
--- a/docs/contributors/runtime.html
+++ b/docs/contributors/runtime.html
@@ -9,22 +9,22 @@
   
 
   
-
+  
   Runtime Phase — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,32 +305,32 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -344,13 +342,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
 
    Runtime Phase
    
 
    The Runtime phase is responsible for constructing self standing TorchScript graphs with embedded TensorRT engines and serving as the runtime
@@ -427,25 +425,25 @@ 
    ABI Versioning and Serialization Format
-
+      
         
-
-
+      
+      
         
-
+      
     
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -453,11 +451,11 @@ 
    ABI Versioning and Serialization FormatSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -484,12 +482,12 @@ 
    ABI Versioning and Serialization Format
          
          
@@ -498,9 +496,9 @@ 
    ABI Versioning and Serialization Format
          
          
+     
 
-
-
+  
 
   
   
@@ -511,7 +509,7 @@ 
    ABI Versioning and Serialization Format
 
   
-
+  
   System Overview — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -34,13 +34,13 @@
     
     
     
-    
+    
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,30 +305,30 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -342,13 +340,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
 
    System Overview
    
 
    Torch-TensorRT is primarily a C++ Library with a Python API planned. We use Bazel as our build system and target Linux x86_64 and
@@ -409,25 +407,25 @@ 
    Compilation and Runtime
-
+             
             
    
             
 
@@ -468,12 +466,12 @@ 
    Compilation and Runtime
     
    
 
+  
 
 
+  
 
-
-
-
+     
        
          
          
@@ -482,9 +480,9 @@ 
    Compilation and Runtime
          
          
+     
 
-
-
+  
 
   
   
@@ -495,7 +493,7 @@ 
    Compilation and Runtime
+   
 
   
 
diff --git a/docs/contributors/useful_links.html b/docs/contributors/useful_links.html
index ddcff90476..fe0dc4fa96 100644
--- a/docs/contributors/useful_links.html
+++ b/docs/contributors/useful_links.html
@@ -9,22 +9,22 @@
   
 
   
-
+  
   Useful Links for Torch-TensorRT Development — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,30 +305,30 @@
 
    
 
   
      
-
+    
       
    • 
         
-
+          
             Docs
-
+          
          >
       
      • 
 
-
+        
       
    • Useful Links for Torch-TensorRT Development
      • 
-
-
+    
+    
       
    • 
-
-
+        
+            
             
-
-
+          
+        
       
      • 
-
+    
   
    
 
-
+  
 
    
         
    
 
@@ -342,13 +340,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
   
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -417,11 +415,11 @@ 
    PyTorch IR DocumentationSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -449,12 +447,12 @@ 
    PyTorch IR Documentation
          
          
@@ -463,9 +461,9 @@ 
    PyTorch IR Documentation
          
          
+     
 
-
-
+  
 
   
   
@@ -476,7 +474,7 @@ 
    PyTorch IR Documentation
 
   
-
+  
   Writing Converters — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -36,11 +36,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -180,9 +180,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -193,21 +193,21 @@
       
    
         
       
    
     
@@ -288,7 +286,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -307,30 +305,30 @@
 
    
 
   
      
-
+    
       
    • 
         
-
+          
             Docs
-
+          
          >
       
      • 
 
-
+        
       
    • Writing Converters
      • 
-
-
+    
+    
       
    • 
-
-
+        
+            
             
-
-
+          
+        
       
      • 
-
+    
   
    
 
-
+  
 
    
         
    
 
@@ -342,13 +340,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
+  
 
-
-
+  
 
     
    
 
-
+  
 
   
    
     
    
@@ -502,11 +500,11 @@ 
    Other adviceSphinx using a theme provided by Read the Docs.
       
    
-
+     
 
 
 
@@ -539,12 +537,12 @@ 
    Other advice
          
          
@@ -553,9 +551,9 @@ 
    Other advice
          
          
+     
 
-
-
+  
 
   
   
@@ -566,7 +564,7 @@ 
    Other advice 
 
   
-
+  
   
-
+  
   Index — Torch-TensorRT master documentation
+  
 
+  
+  
+  
+  
 
+  
 
+  
+  
+    
 
-
-
-
-
-
-
-
-
-
-
+  
 
   
   
@@ -33,11 +33,11 @@
     
     
   
-
+  
   
+  
 
-
-
+  
   
 
   
@@ -177,9 +177,9 @@
 
 
 
+   
 
-
-
+    
 
     
    
       Table of Contents
@@ -190,21 +190,21 @@
       
    
         
       
    
     
@@ -285,7 +283,7 @@
     
    
       
    
         
    
-
+          
 
 
 
@@ -304,28 +302,28 @@
 
    
 
   
 
-
+  
 
    
         
    
 
@@ -337,13 +335,13 @@
       
    
         
    
 
-
-
+        
+          
           
    
-
+          
             
    
              
    
-
+              
 
 
    Index
    
 
@@ -364,7 +362,7 @@ 
    Index
    
  | T
  | W
  | X
-
+ 
 
    
 
    _
    
 
@@ -510,10 +508,12 @@ 
    I
  • info (class in torch_tensorrt.logging)
 
  • Info (torch_tensorrt.logging.Level attribute)
+
    • 
+      
  • Input (class in torch_tensorrt)
 
    • 
       
       
   
   
 
    
@@ -902,16 +927,16 @@ 
    X

    - +
    @@ -933,19 +958,19 @@

    X

    - +
    + + - - - + @@ -954,9 +979,9 @@

    X

    + - - + @@ -967,7 +992,7 @@

    X

    jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/getting_started/getting_started_with_cpp_api.html b/docs/getting_started/getting_started_with_cpp_api.html new file mode 100644 index 0000000000..7b7b4cce68 --- /dev/null +++ b/docs/getting_started/getting_started_with_cpp_api.html @@ -0,0 +1,935 @@ + + + + + + + + + + + + + Using Torch-TensorRT in C++ — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + +
      + +
    • + + + Docs + + > +
      • + + +
    • Using Torch-TensorRT in C++
      • + + +
    • + + + + + +
      • + +
    + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Using Torch-TensorRT in C++

    +

    If you haven’t already, acquire a tarball of the library by following the instructions in Installation

    +
    +

    Using Torch-TensorRT in C++

    +

    Torch-TensorRT C++ API accepts TorchScript modules (generated either from torch.jit.script or torch.jit.trace) as an input and returns +a Torchscript module (optimized using TensorRT). This requires users to use Pytorch (in python) to generate torchscript modules beforehand. +Please refer to Creating TorchScript modules in Python section to generate torchscript graphs.

    +
    +

    [Torch-TensorRT Quickstart] Compiling TorchScript Modules with torchtrtc

    +

    An easy way to get started with Torch-TensorRT and to check if your model can be supported without extra work is to run it through +torchtrtc, which supports almost all features of the compiler from the command line including post training quantization +(given a previously created calibration cache). For example we can compile our lenet model by setting our preferred operating +precision and input size. This new TorchScript file can be loaded into Python (note: you need to import torch_tensorrt before loading +these compiled modules because the compiler extends the PyTorch the deserializer and runtime to execute compiled modules).

    +
    ❯ torchtrtc -p f16 lenet_scripted.ts trt_lenet_scripted.ts "(1,1,32,32)"
+
+❯ python3
+Python 3.6.9 (default, Apr 18 2020, 01:56:04)
+[GCC 8.4.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import torch
+>>> import torch_tensorrt
+>>> ts_model = torch.jit.load(“trt_lenet_scripted.ts”)
+>>> ts_model(torch.randn((1,1,32,32)).to(“cuda”).half())
+
    +
    +

    You can learn more about torchtrtc usage here: torchtrtc

    +
    +
    +

    Working with TorchScript in C++

    +

    If we are developing an application to deploy with C++, we can save either our traced or scripted module using torch.jit.save +which will serialize the TorchScript code, weights and other information into a package. This is also where our dependency on Python ends.

    +
    torch_script_module.save("lenet.jit.pt")
+
    +
    +

    From here we can now load our TorchScript module in C++

    +
    #include <torch/script.h> // One-stop header.
+
+#include <iostream>
+#include <memory>
+
+int main(int argc, const char* argv[]) {
+    torch::jit::Module module;
+    try {
+        // Deserialize the ScriptModule from a file using torch::jit::load().
+        module = torch::jit::load("<PATH TO SAVED TS MOD>");
+    }
+    catch (const c10::Error& e) {
+        std::cerr << "error loading the model\n";
+        return -1;
+    }
+
+    std::cout << "ok\n";
+
    +
    +

    You can do full training and inference in C++ with PyTorch / LibTorch if you would like, you can even define your modules in C++ and +have access to the same powerful tensor library that backs PyTorch. (For more information: https://pytorch.org/cppdocs/). +For instance we can do inference with our LeNet module like this:

    +
    mod.eval();
+torch::Tensor in = torch::randn({1, 1, 32, 32});
+auto out = mod.forward(in);
+
    +
    +

    and to run on the GPU:

    +
    mod.eval();
+mod.to(torch::kCUDA);
+torch::Tensor in = torch::randn({1, 1, 32, 32}, torch::kCUDA);
+auto out = mod.forward(in);
+
    +
    +

    As you can see it is pretty similar to the Python API. When you call the forward method, you invoke the PyTorch JIT compiler, which will optimize and run your TorchScript code.

    +
    +
    +

    Compiling with Torch-TensorRT in C++

    +

    We are also at the point were we can compile and optimize our module with Torch-TensorRT, but instead of in a JIT fashion we must do it ahead-of-time (AOT) i.e. before we start doing actual inference work +since it takes a bit of time to optimize the module, it would not make sense to do this every time you run the module or even the first time you run it.

    +

    With our module loaded, we can feed it into the Torch-TensorRT compiler. When we do so we must provide some information on the expected input size and also configure any additional settings.

    +
    #include "torch/script.h"
+#include "torch_tensorrt/torch_tensorrt.h"
+...
+
+    mod.to(at::kCUDA);
+    mod.eval();
+
+    auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA});
+    auto trt_mod = torch_tensorrt::CompileGraph(mod, std::vector<torch_tensorrt::CompileSpec::InputRange>{{in.sizes()}});
+    auto out = trt_mod.forward({in});
+
    +
    +

    Thats it! Now the graph runs primarily not with the JIT compiler but using TensorRT (though we execute the graph using the JIT runtime).

    +

    We can also set settings like operating precision to run in FP16.

    +
    #include "torch/script.h"
+#include "torch_tensorrt/torch_tensorrt.h"
+...
+
+    mod.to(at::kCUDA);
+    mod.eval();
+
+    auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA}).to(torch::kHALF);
+    auto input_sizes = std::vector<torch_tensorrt::CompileSpec::InputRange>({in.sizes()});
+    torch_tensorrt::CompileSpec info(input_sizes);
+    info.enable_precisions.insert(torch::kHALF);
+    auto trt_mod = torch_tensorrt::CompileGraph(mod, info);
+    auto out = trt_mod.forward({in});
+
    +
    +

    And now we are running the module in FP16 precision. You can then save the module to load later.

    +
    trt_mod.save("<PATH TO SAVED TRT/TS MOD>")
+
    +
    +

    Torch-TensorRT compiled TorchScript modules are loaded in the same way as normal TorchScript module. Make sure your deployment application is linked against libtorchtrt.so

    +
    #include "torch/script.h"
+#include "torch_tensorrt/torch_tensorrt.h"
+
+int main(int argc, const char* argv[]) {
+    torch::jit::Module module;
+    try {
+        // Deserialize the ScriptModule from a file using torch::jit::load().
+        module = torch::jit::load("<PATH TO SAVED TRT/TS MOD>");
+    }
+    catch (const c10::Error& e) {
+        std::cerr << "error loading the model\n";
+        return -1;
+    }
+
+    torch::Tensor in = torch::randn({1, 1, 32, 32}, torch::kCUDA);
+    auto out = mod.forward(in);
+
+    std::cout << "ok\n";
+}
+
    +
    +

    If you want to save the engine produced by Torch-TensorRT to use in a TensorRT application you can use the ConvertGraphToTRTEngine API.

    +
    #include "torch/script.h"
+#include "torch_tensorrt/torch_tensorrt.h"
+...
+
+    mod.to(at::kCUDA);
+    mod.eval();
+
+    auto in = torch::randn({1, 1, 32, 32}, {torch::kCUDA}).to(torch::kHALF);
+    auto input_sizes = std::vector<torch_tensorrt::CompileSpec::InputRange>({in.sizes()});
+    torch_tensorrt::CompileSpec info(input_sizes);
+    info.enabled_precisions.insert(torch::kHALF);
+    auto trt_mod = torch_tensorrt::ConvertGraphToTRTEngine(mod, "forward", info);
+    std::ofstream out("/tmp/engine_converted_from_jit.trt");
+    out << engine;
+    out.close();
+
    +
    +
    +
    +

    Under The Hood

    +

    When a module is provided to Torch-TensorRT, the compiler starts by mapping a graph like you saw above to a graph like this:

    +
    graph(%input.2 : Tensor):
+    %2 : Float(84, 10) = prim::Constant[value=<Tensor>]()
+    %3 : Float(120, 84) = prim::Constant[value=<Tensor>]()
+    %4 : Float(576, 120) = prim::Constant[value=<Tensor>]()
+    %5 : int = prim::Constant[value=-1]() # x.py:25:0
+    %6 : int[] = prim::Constant[value=annotate(List[int], [])]()
+    %7 : int[] = prim::Constant[value=[2, 2]]()
+    %8 : int[] = prim::Constant[value=[0, 0]]()
+    %9 : int[] = prim::Constant[value=[1, 1]]()
+    %10 : bool = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+    %11 : int = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+    %12 : bool = prim::Constant[value=0]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+    %self.classifer.fc3.bias : Float(10) = prim::Constant[value= 0.0464  0.0383  0.0678  0.0932  0.1045 -0.0805 -0.0435 -0.0818  0.0208 -0.0358 [ CUDAFloatType{10} ]]()
+    %self.classifer.fc2.bias : Float(84) = prim::Constant[value=<Tensor>]()
+    %self.classifer.fc1.bias : Float(120) = prim::Constant[value=<Tensor>]()
+    %self.feat.conv2.weight : Float(16, 6, 3, 3) = prim::Constant[value=<Tensor>]()
+    %self.feat.conv2.bias : Float(16) = prim::Constant[value=<Tensor>]()
+    %self.feat.conv1.weight : Float(6, 1, 3, 3) = prim::Constant[value=<Tensor>]()
+    %self.feat.conv1.bias : Float(6) = prim::Constant[value= 0.0530 -0.1691  0.2802  0.1502  0.1056 -0.1549 [ CUDAFloatType{6} ]]()
+    %input0.4 : Tensor = aten::_convolution(%input.2, %self.feat.conv1.weight, %self.feat.conv1.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+    %input0.5 : Tensor = aten::relu(%input0.4) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+    %input1.2 : Tensor = aten::max_pool2d(%input0.5, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+    %input0.6 : Tensor = aten::_convolution(%input1.2, %self.feat.conv2.weight, %self.feat.conv2.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
+    %input2.1 : Tensor = aten::relu(%input0.6) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+    %x.1 : Tensor = aten::max_pool2d(%input2.1, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
+    %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0
+    %27 : Tensor = aten::matmul(%input.1, %4)
+    %28 : Tensor = trt::const(%self.classifer.fc1.bias)
+    %29 : Tensor = aten::add_(%28, %27, %11)
+    %input0.2 : Tensor = aten::relu(%29) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+    %31 : Tensor = aten::matmul(%input0.2, %3)
+    %32 : Tensor = trt::const(%self.classifer.fc2.bias)
+    %33 : Tensor = aten::add_(%32, %31, %11)
+    %input1.1 : Tensor = aten::relu(%33) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
+    %35 : Tensor = aten::matmul(%input1.1, %2)
+    %36 : Tensor = trt::const(%self.classifer.fc3.bias)
+    %37 : Tensor = aten::add_(%36, %35, %11)
+    return (%37)
+(CompileGraph)
+
    +
    +

    The graph has now been transformed from a collection of modules, each managing their own parameters into a single graph with the parameters inlined +into the graph and all of the operations laid out. Torch-TensorRT has also executed a number of optimizations and mappings to make the graph easier to translate to TensorRT. +From here the compiler can assemble the TensorRT engine by following the dataflow through the graph.

    +

    When the graph construction phase is complete, Torch-TensorRT produces a serialized TensorRT engine. From here depending on the API, this engine is returned +to the user or moves into the graph construction phase. Here Torch-TensorRT creates a JIT Module to execute the TensorRT engine which will be instantiated and managed +by the Torch-TensorRT runtime.

    +

    Here is the graph that you get back after compilation is complete:

    +
    graph(%self_1 : __torch__.lenet, %input_0 : Tensor):
+    %1 : ...trt.Engine = prim::GetAttr[name="lenet"](%self_1)
+    %3 : Tensor[] = prim::ListConstruct(%input_0)
+    %4 : Tensor[] = trt::execute_engine(%3, %1)
+    %5 : Tensor = prim::ListUnpack(%4)
+    return (%5)
+
    +
    +

    You can see the call where the engine is executed, after extracting the attribute containing the engine and constructing a list of inputs, then returns the tensors back to the user.

    +
    +
    +

    Working with Unsupported Operators

    +

    Torch-TensorRT is a new library and the PyTorch operator library is quite large, so there will be ops that aren’t supported natively by the compiler. You can either use the composition techinques +shown above to make modules are fully Torch-TensorRT supported and ones that are not and stitch the modules together in the deployment application or you can register converters for missing ops.

    +
    +

    You can check support without going through the full compilation pipleine using the torch_tensorrt::CheckMethodOperatorSupport(const torch::jit::Module& module, std::string method_name) api +to see what operators are not supported. torchtrtc automatically checks modules with this method before starting compilation and will print out a list of operators that are not supported.

    +
    +
    +

    Registering Custom Converters

    +

    Operations are mapped to TensorRT through the use of modular converters, a function that takes a node from a the JIT graph and produces an equivalent layer or subgraph in TensorRT. +Torch-TensorRT ships with a library of these converters stored in a registry, that will be executed depending on the node being parsed. For instance a aten::relu(%input0.4) instruction will trigger +the relu converter to be run on it, producing an activation layer in the TensorRT graph. But since this library is not exhaustive you may need to write your own to get Torch-TensorRT +to support your module.

    +

    Shipped with the Torch-TensorRT distribution are the internal core API headers. You can therefore access the converter registry and add a converter for the op you need.

    +

    For example, if we try to compile a graph with a build of Torch-TensorRT that doesn’t support the flatten operation (aten::flatten) you may see this error:

    +
    terminate called after throwing an instance of 'torch_tensorrt::Error'
+what():  [enforce fail at core/conversion/conversion.cpp:109] Expected converter to be true but got false
+Unable to convert node: %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0 (conversion.AddLayer)
+Schema: aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor)
+Converter for aten::flatten requested, but no such converter was found.
+If you need a converter for this operator, you can try implementing one yourself
+or request a converter: https://www.github.com/NVIDIA/Torch-TensorRT/issues
+
    +
    +

    We can register a converter for this operator in our application. All of the tools required to build a converter can be imported by including torch_tensorrt/core/conversion/converters/converters.h. +We start by creating an instance of the self-registering class torch_tensorrt::core::conversion::converters::RegisterNodeConversionPatterns() which will register converters +in the global converter registry, associating a function schema like aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor) with a lambda that +will take the state of the conversion, the node/operation in question to convert and all of the inputs to the node and produces as a side effect a new layer in the TensorRT network. +Arguments are passed as a vector of inspectable unions of TensorRT ITensors and Torch IValues in the order arguments are listed in the schema.

    +

    Below is a implementation of a aten::flatten converter that we can use in our application. You have full access to the Torch and TensorRT libraries in the converter implementation. So +for example we can quickly get the output size by just running the operation in PyTorch instead of implementing the full calculation outself like we do below for this flatten converter.

    +
    #include "torch/script.h"
+#include "torch_tensorrt/torch_tensorrt.h"
+#include "torch_tensorrt/core/conversion/converters/converters.h"
+
+static auto flatten_converter = torch_tensorrt::core::conversion::converters::RegisterNodeConversionPatterns()
+    .pattern({
+        "aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -> (Tensor)",
+        [](torch_tensorrt::core::conversion::ConversionCtx* ctx,
+           const torch::jit::Node* n,
+           torch_tensorrt::core::conversion::converters::args& args) -> bool {
+            auto in = args[0].ITensor();
+            auto start_dim = args[1].unwrapToInt();
+            auto end_dim = args[2].unwrapToInt();
+            auto in_shape = torch_tensorrt::core::util::toVec(in->getDimensions());
+            auto out_shape = torch::flatten(torch::rand(in_shape), start_dim, end_dim).sizes();
+
+            auto shuffle = ctx->net->addShuffle(*in);
+            shuffle->setReshapeDimensions(torch_tensorrt::core::util::toDims(out_shape));
+            shuffle->setName(torch_tensorrt::core::util::node_info(n).c_str());
+
+            auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], shuffle->getOutput(0));
+            return true;
+        }
+    });
+
+int main() {
+    ...
+
    +
    +

    To use this converter in Python, it is recommended to use PyTorch’s C++ / CUDA Extention +template to wrap your library of converters into a .so that you can load with ctypes.CDLL() in your Python application.

    +

    You can find more information on all the details of writing converters in the contributors documentation (Writing Converters). +If you find yourself with a large library of converter implementations, do consider upstreaming them, PRs are welcome and it would be great for the community to benefit as well.

    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/getting_started/getting_started_with_python_api.html b/docs/getting_started/getting_started_with_python_api.html new file mode 100644 index 0000000000..d33aefeefa --- /dev/null +++ b/docs/getting_started/getting_started_with_python_api.html @@ -0,0 +1,689 @@ + + + + + + + + + + + + + Using Torch-TensorRT in Python — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + +
      + +
    • + + + Docs + + > +
      • + + +
    • Using Torch-TensorRT in Python
      • + + +
    • + + + + + +
      • + +
    + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Using Torch-TensorRT in Python

    +

    The Torch-TensorRT Python API supports a number of unique usecases compared to the CLI and C++ APIs which solely support TorchScript compilation.

    +

    Torch-TensorRT Python API can accept a torch.nn.Module, torch.jit.ScriptModule, or torch.fx.GraphModule as an input. +Depending on what is provided one of the two frontends (TorchScript or FX) will be selected to compile the module. Provided the +module type is supported, users may explicitly set which frontend they would like to use using the ir flag for compile. +If given a torch.nn.Module and the ir flag is set to either default or torchscript the module will be run through +torch.jit.script to convert the input module into a TorchScript module.

    +

    To compile your input torch.nn.Module with Torch-TensorRT, all you need to do is provide the module and inputs +to Torch-TensorRT and you will be returned an optimized TorchScript module to run or add into another PyTorch module. Inputs +is a list of torch_tensorrt.Input classes which define input’s shape, datatype and memory format. You can also specify settings such as +operating precision for the engine or target device. After compilation you can save the module just like any other module +to load in a deployment application. In order to load a TensorRT/TorchScript module, make sure you first import torch_tensorrt.

    +
    import torch_tensorrt
+
+...
+
+model = MyModel().eval()  # torch module needs to be in eval (not training) mode
+
+inputs = [
+    torch_tensorrt.Input(
+        min_shape=[1, 1, 16, 16],
+        opt_shape=[1, 1, 32, 32],
+        max_shape=[1, 1, 64, 64],
+        dtype=torch.half,
+    )
+]
+enabled_precisions = {torch.float, torch.half}  # Run with fp16
+
+trt_ts_module = torch_tensorrt.compile(
+    model, inputs=inputs, enabled_precisions=enabled_precisions
+)
+
+input_data = input_data.to("cuda").half()
+result = trt_ts_module(input_data)
+torch.jit.save(trt_ts_module, "trt_ts_module.ts")
+
    +
    +
    # Deployment application
+import torch
+import torch_tensorrt
+
+trt_ts_module = torch.jit.load("trt_ts_module.ts")
+input_data = input_data.to("cuda").half()
+result = trt_ts_module(input_data)
+
    +
    +

    Torch-TensorRT Python API also provides torch_tensorrt.ts.compile which accepts a TorchScript module as input and torch_tensorrt.fx.compile which accepts a FX GraphModule as input.

    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/getting_started/installation.html b/docs/getting_started/installation.html new file mode 100644 index 0000000000..e99c84606d --- /dev/null +++ b/docs/getting_started/installation.html @@ -0,0 +1,1012 @@ + + + + + + + + + + + + + Installation — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + + + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Installation

    +
    +

    Precompiled Binaries

    +
    +

    Dependencies

    +

    You need to have either PyTorch or LibTorch installed based on if you are using Python or C++ +and you must have CUDA, cuDNN and TensorRT installed.

    +
    +
    +
    +
    +

    Python Package

    +

    You can install the python package using

    +
    pip3 install torch-tensorrt -f https://github.com/pytorch/TensorRT/releases
+
    +
    +
    +
    +

    C++ Binary Distribution

    +

    Precompiled tarballs for releases are provided here: https://github.com/pytorch/TensorRT/releases

    +
    +
    +
    +

    Compiling From Source

    +
    +

    Dependencies for Compilation

    +

    Torch-TensorRT is built with Bazel, so begin by installing it.

    +
    +
    +
    export BAZEL_VERSION=$(cat <PATH_TO_TORCHTRT_ROOT>/.bazelversion)
+mkdir bazel
+cd bazel
+curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip
+unzip bazel-$BAZEL_VERSION-dist.zip
+bash ./compile.sh
+cp output/bazel /usr/local/bin/
+
    +
    +
    +

    You will also need to have CUDA installed on the system (or if running in a container, the system must have +the CUDA driver installed and the container must have CUDA)

    +

    The correct LibTorch version will be pulled down for you by bazel.

    +
    +

    NOTE: For best compatability with official PyTorch, use torch==1.10.0+cuda113, TensorRT 8.0 and cuDNN 8.2 for CUDA 11.3 however Torch-TensorRT itself supports +TensorRT and cuDNN for other CUDA versions for usecases such as using NVIDIA compiled distributions of PyTorch that use other versions of CUDA +e.g. aarch64 or custom compiled version of PyTorch.

    +
    +
    +

    Choosing the Right ABI

    +

    Likely the most complicated thing about compiling Torch-TensorRT is selecting the correct ABI. There are two options +which are incompatible with each other, pre-cxx11-abi and the cxx11-abi. The complexity comes from the fact that while +the most popular distribution of PyTorch (wheels downloaded from pytorch.org/pypi directly) use the pre-cxx11-abi, most +other distributions you might encounter (e.g. ones from NVIDIA - NGC containers, and builds for Jetson as well as certain +libtorch builds and likely if you build PyTorch from source) use the cxx11-abi. It is important you compile Torch-TensorRT +using the correct ABI to function properly. Below is a table with general pairings of PyTorch distribution sources and the +recommended commands:

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    PyTorch Source

    Recommended C++ Compilation Command

    Recommended Python Compilation Command

    PyTorch whl file from PyTorch.org

    bazel build //:libtorchtrt -c opt –config pre_cxx11_abi

    python3 setup.py bdist_wheel

    libtorch-shared-with-deps-*.zip from PyTorch.org

    bazel build //:libtorchtrt -c opt –config pre_cxx11_abi

    python3 setup.py bdist_wheel

    libtorch-cxx11-abi-shared-with-deps-*.zip from PyTorch.org

    bazel build //:libtorchtrt -c opt

    python3 setup.py bdist_wheel –use-cxx11-abi

    PyTorch preinstalled in an NGC container

    bazel build //:libtorchtrt -c opt

    python3 setup.py bdist_wheel –use-cxx11-abi

    PyTorch from the NVIDIA Forums for Jetson

    bazel build //:libtorchtrt -c opt

    python3 setup.py bdist_wheel –jetpack-version 4.6 –use-cxx11-abi

    PyTorch built from Source

    bazel build //:libtorchtrt -c opt

    python3 setup.py bdist_wheel –use-cxx11-abi

    +
    +

    NOTE: For all of the above cases you must correctly declare the source of PyTorch you intend to use in your WORKSPACE file for both Python and C++ builds. See below for more information

    +
    +

    You then have two compilation options:

    +
    +
    +
    +

    Building using cuDNN & TensorRT tarball distributions

    +
    +

    This is recommended so as to build Torch-TensorRT hermetically and insures any compilation errors are not caused by version issues

    +

    Make sure when running Torch-TensorRT that these versions of the libraries are prioritized in your $LD_LIBRARY_PATH

    +
    +
    +
    You need to download the tarball distributions of TensorRT and cuDNN from the NVIDIA website.
    +
    +
    +

    Place these files in a directory (the directories third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu] exist for this purpose)

    +

    Then compile referencing the directory with the tarballs

    +
    +

    If you get errors regarding the packages, check their sha256 hashes and make sure they match the ones listed in WORKSPACE

    +
    +
    +

    Release Build

    +
    bazel build //:libtorchtrt -c opt --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
    +
    +

    A tarball with the include files and library can then be found in bazel-bin

    +
    +
    +

    Debug Build

    +

    To build with debug symbols use the following command

    +
    bazel build //:libtorchtrt -c dbg --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
    +
    +

    A tarball with the include files and library can then be found in bazel-bin

    +
    +
    +

    Pre CXX11 ABI Build

    +

    To build using the pre-CXX11 ABI use the pre_cxx11_abi config

    +
    bazel build //:libtorchtrt --config pre_cxx11_abi -c [dbg/opt] --distdir third_party/distdir/[x86_64-linux-gnu | aarch64-linux-gnu]
+
    +
    +

    A tarball with the include files and library can then be found in bazel-bin

    +
    +
    +
    +

    Building using locally installed cuDNN & TensorRT

    +
    +

    If you encounter bugs and you compiled using this method please disclose that you used local sources in the issue (an ldd dump would be nice too)

    +
    +

    Install TensorRT, CUDA and cuDNN on the system before starting to compile.

    +

    In WORKSPACE comment out:

    +
    # Downloaded distributions to use with --distdir
+http_archive(
+    name="cudnn",
+    urls=[
+        "<URL>",
+    ],
+    build_file="@//third_party/cudnn/archive:BUILD",
+    sha256="<TAR SHA256>",
+    strip_prefix="cuda",
+)
+
+http_archive(
+    name="tensorrt",
+    urls=[
+        "<URL>",
+    ],
+    build_file="@//third_party/tensorrt/archive:BUILD",
+    sha256="<TAR SHA256>",
+    strip_prefix="TensorRT-<VERSION>",
+)
+
    +
    +

    and uncomment

    +
    # Locally installed dependencies
+new_local_repository(
+    name="cudnn", path="/usr/", build_file="@//third_party/cudnn/local:BUILD"
+)
+
+new_local_repository(
+    name="tensorrt", path="/usr/", build_file="@//third_party/tensorrt/local:BUILD"
+)
+
    +
    +
    +

    Release Build

    +

    Compile using:

    +
    bazel build //:libtorchtrt -c opt
+
    +
    +

    A tarball with the include files and library can then be found in bazel-bin

    +
    +
    +

    Debug Build

    +

    To build with debug symbols use the following command

    +
    bazel build //:libtorchtrt -c dbg
+
    +
    +

    A tarball with the include files and library can then be found in bazel-bin

    +
    +
    +

    Pre CXX11 ABI Build

    +

    To build using the pre-CXX11 ABI use the pre_cxx11_abi config

    +
    bazel build //:libtorchtrt --config pre_cxx11_abi -c [dbg/opt]
+
    +
    +
    +
    +
    +

    Building with CMake

    +

    It is possible to build the API libraries (in cpp/) and the torchtrtc executable using CMake instead of Bazel. +Currently, the python API and the tests cannot be built with CMake. +Begin by installing CMake.

    +
    +
      +

    • Latest releases of CMake and instructions on how to install are available for different platforms +[on their website](https://cmake.org/download/).

      • +
    +
    +

    A few useful CMake options include:

    +
    +
      +

    • CMake finders for TensorRT and cuDNN are provided in cmake/Modules. In order for CMake to use them, pass +-DCMAKE_MODULE_PATH=cmake/Modules when configuring the project with CMake.

      • +

    • Libtorch provides its own CMake finder. In case CMake doesn’t find it, pass the path to your install of +libtorch with -DTorch_DIR=<path to libtorch>/share/cmake/Torch

      • +

    • If TensorRT is not found with the provided cmake finder, specify -DTensorRT_ROOT=<path to TensorRT>

      • +

    • Finally, configure and build the project in a build directory of your choice with the following command +from the root of Torch-TensorRT project:

      • +
    +
    cmake -S. -B<build directory> \
+    [-DCMAKE_MODULE_PATH=cmake/Module] \
+    [-DTorch_DIR=<path to libtorch>/share/cmake/Torch] \
+    [-DTensorRT_ROOT=<path to TensorRT>] \
+    [-DCMAKE_BUILD_TYPE=Debug|Release]
+cmake --build <build directory>
+
    +
    +
    +
    +
    +

    Building the Python package

    +

    Begin by installing ninja

    +

    You can build the Python package using setup.py (this will also build the correct version of libtorchtrt.so)

    +
    python3 setup.py [install/bdist_wheel]
+
    +
    +
    +

    Debug Build

    +
    python3 setup.py develop [--user]
+
    +
    +

    This also compiles a debug build of libtorchtrt.so

    +
    +
    +
    +

    Building Natively on aarch64 (Jetson)

    +
    +

    Prerequisites

    +

    Install or compile a build of PyTorch/LibTorch for aarch64

    +

    NVIDIA hosts builds the latest release branch for Jetson here:

    +
    +
    +
    +
    +

    Enviorment Setup

    +

    To build natively on aarch64-linux-gnu platform, configure the WORKSPACE with local available dependencies.

    +
      +

    1. Replace WORKSPACE with the corresponding WORKSPACE file in //toolchains/jp_workspaces

      2. +

    2. Configure the correct paths to directory roots containing local dependencies in the new_local_repository rules:

      +
      +

      NOTE: If you installed PyTorch using a pip package, the correct path is the path to the root of the python torch package. +In the case that you installed with sudo pip install this will be /usr/local/lib/python3.8/dist-packages/torch. +In the case you installed with pip install --user this will be $HOME/.local/lib/python3.8/site-packages/torch.

      +
      +
      3. +
    +

    In the case you are using NVIDIA compiled pip packages, set the path for both libtorch sources to the same path. This is because unlike +PyTorch on x86_64, NVIDIA aarch64 PyTorch uses the CXX11-ABI. If you compiled for source using the pre_cxx11_abi and only would like to +use that library, set the paths to the same path but when you compile make sure to add the flag --config=pre_cxx11_abi

    +
    new_local_repository(
+    name = "libtorch",
+    path = "/usr/local/lib/python3.8/dist-packages/torch",
+    build_file = "third_party/libtorch/BUILD"
+)
+
+new_local_repository(
+    name = "libtorch_pre_cxx11_abi",
+    path = "/usr/local/lib/python3.8/dist-packages/torch",
+    build_file = "third_party/libtorch/BUILD"
+)
+
    +
    +
    +
    +

    Compile C++ Library and Compiler CLI

    +
    +

    NOTE: Due to shifting dependency locations between Jetpack 4.5 and 4.6 there is a now a flag to inform bazel of the Jetpack version

    +
    --platforms //toolchains:jetpack_x.x
+
    +
    +
    +

    Compile Torch-TensorRT library using bazel command:

    +
    bazel build //:libtorchtrt --platforms //toolchains:jetpack_5.0
+
    +
    +
    +
    +

    Compile Python API

    +
    +

    NOTE: Due to shifting dependencies locations between Jetpack 4.5 and newer Jetpack verisons there is now a flag for setup.py which sets the jetpack version (default: 5.0)

    +
    +

    Compile the Python API using the following command from the //py directory:

    +
    python3 setup.py install --use-cxx11-abi
+
    +
    +

    If you have a build of PyTorch that uses Pre-CXX11 ABI drop the --use-cxx11-abi flag

    +

    If you are building for Jetpack 4.5 add the --jetpack-version 5.0 flag

    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index bd2e17d021..1b468953f5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,22 +9,22 @@ - + Torch-TensorRT — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,13 +33,13 @@ - + - + + - - + @@ -179,9 +179,9 @@ + - - +
    Table of Contents @@ -192,21 +192,21 @@
    @@ -287,7 +285,7 @@
    - + @@ -306,30 +304,30 @@
    - +
    @@ -341,13 +339,13 @@
    - - + +
    - +
    - +

    Torch-TensorRT

    @@ -367,19 +365,27 @@

    Ahead-of-time compilation of TorchScript / PyTorch JIT for NVIDIA GPUs

    Getting Started

    +
    +
    +

    +
    +

    Tutorials

    +
    -
    -

    Python API Documenation

    @@ -388,6 +394,7 @@

    Python API Documenation

    torch_tensorrt.logging

  • torch_tensorrt.ptq

  • torch_tensorrt.ts

    • +

  • torch_tensorrt.fx

    • @@ -403,6 +410,14 @@

    C++ API Documenation

    +
    +

    CLI Documentation

    + +
    +
    +

    Contributor Documentation

    + - - +
    - +

    @@ -451,11 +466,11 @@

    IndicesSphinx using a theme provided by Read the Docs.

    - + @@ -468,11 +483,11 @@

    IndicesTorch-TensorRT @@ -485,12 +500,12 @@

    Indices @@ -499,9 +514,9 @@

    Indices + - - + @@ -512,7 +527,7 @@

    Indices - + Operators Supported — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -35,11 +35,11 @@ - + + - - + @@ -179,9 +179,9 @@ + - - +
    Table of Contents @@ -192,21 +192,21 @@
    @@ -287,7 +285,7 @@
    - + @@ -306,30 +304,30 @@
      - +
    • - + Docs - + >
      • - +
    • Operators Supported
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -341,13 +339,13 @@
    - - + +
    - +
    - +

    Operators Supported

    @@ -632,23 +630,23 @@

    Operators Currently Supported Through Evaluators - - + + - +

    + - - +
    - +

    @@ -656,11 +654,11 @@

    Operators Currently Supported Through EvaluatorsSphinx using a theme provided by Read the Docs.

    - + @@ -684,12 +682,12 @@

    Operators Currently Supported Through Evaluators @@ -698,9 +696,9 @@

    Operators Currently Supported Through Evaluators + - - + @@ -711,7 +709,7 @@

    Operators Currently Supported Through Evaluators - + - + Python Module Index — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,13 +34,13 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,26 +305,26 @@
    - +
    @@ -338,13 +336,13 @@
    - - + +
    - +
    - +

    Python Module Index

    @@ -362,6 +360,11 @@

    Python Module Index

    torch_tensorrt + + +     + torch_tensorrt.fx +     @@ -381,16 +384,16 @@

    Python Module Index

    - +
    + - - +
    - +

    @@ -398,11 +401,11 @@

    Python Module Index

    - +
    Built with Sphinx using a theme provided by Read the Docs.
    - +
    @@ -412,19 +415,19 @@

    Python Module Index

    - +
    + + - - - + @@ -433,9 +436,9 @@

    Python Module Index

    + - - + @@ -446,7 +449,7 @@

    Python Module Index

    jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/py_api/fx.html b/docs/py_api/fx.html new file mode 100644 index 0000000000..7367f08c0a --- /dev/null +++ b/docs/py_api/fx.html @@ -0,0 +1,691 @@ + + + + + + + + + + + + + torch_tensorrt.fx — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + + + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    torch_tensorrt.fx

    +
    +

    Functions

    +
    +
    +

    Classes

    +
    +
    +class torch_tensorrt.fx.TRTModule(engine=None, input_names=None, output_names=None, cuda_graph_batch_size=- 1)[source]
    +
    + +
    +
    +class torch_tensorrt.fx.InputTensorSpec(shape: Sequence[int], dtype: torch.dtype, device: torch.device = device(type='cpu'), shape_ranges: List[Tuple[Sequence[int], Sequence[int], Sequence[int]]] = [], has_batch_dim: bool = True)[source]
    +

    This class contains the information of a input tensor.

    +

    shape: shape of the tensor.

    +

    dtype: dtyep of the tensor.

    +
    +
    device: device of the tensor. This is only used to generate inputs to the given model

    in order to run shape prop. For TensorRT engine, inputs have to be on cuda device.

    +
    +
    shape_ranges: If dynamic shape is needed (shape has dimensions of -1), then this field

    has to be provided (default is empty list). Every shape_range is a tuple of three +tuples ((min_input_shape), (optimized_input_shape), (max_input_shape)). Each shape_range +is used to populate a TensorRT optimization profile. +e.g. If the input shape varies from (1, 224) to (100, 224) and we want to optimize +for (25, 224) because it’s the most common input shape, then we set shape_ranges to +((1, 224), (25, 225), (100, 224)).

    +
    +
    has_batch_dim: Whether the shape includes batch dimension. Batch dimension has to be provided

    if the engine want to run with dynamic shape.

    +
    +
    +
    + +
    +
    +class torch_tensorrt.fx.TRTInterpreter(module: torch.fx.graph_module.GraphModule, input_specs: List[torch_tensorrt.fx.input_tensor_spec.InputTensorSpec], explicit_batch_dimension: bool = False, explicit_precision: bool = False, logger_level=None)[source]
    +
    + +
    +
    +class torch_tensorrt.fx.TRTInterpreterResult(engine, input_names, output_names, serialized_cache)[source]
    +
    + +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/py_api/logging.html b/docs/py_api/logging.html index 55eaf6bce1..66556b861a 100644 --- a/docs/py_api/logging.html +++ b/docs/py_api/logging.html @@ -9,22 +9,22 @@ - + torch_tensorrt.logging — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    torch_tensorrt.logging

    @@ -558,25 +556,25 @@
    - +
    @@ -608,12 +606,12 @@
    + + - - - + @@ -622,9 +620,9 @@ + - - + @@ -635,7 +633,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/py_api/ptq.html b/docs/py_api/ptq.html index f085251114..146bc1ac77 100644 --- a/docs/py_api/ptq.html +++ b/docs/py_api/ptq.html @@ -9,22 +9,22 @@ - + torch_tensorrt.ptq — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,13 +342,13 @@
    - - + +
    - +
    - +

    torch_tensorrt.ptq

    @@ -473,25 +471,25 @@

    Enums

    - +
    @@ -527,12 +525,12 @@

    Enums

    + + - - - + @@ -541,9 +539,9 @@

    Enums + - - + @@ -554,7 +552,7 @@

    Enums jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/py_api/torch_tensorrt.html b/docs/py_api/torch_tensorrt.html index 0b3d564c05..9e6947626b 100644 --- a/docs/py_api/torch_tensorrt.html +++ b/docs/py_api/torch_tensorrt.html @@ -9,22 +9,22 @@ - + torch_tensorrt — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,13 +34,13 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    @@ -710,25 +709,25 @@

    Submodules - + - - - - + + + +

    + - - +
    - +

    @@ -736,11 +735,11 @@

    SubmodulesSphinx using a theme provided by Read the Docs.

    - + @@ -766,12 +765,12 @@

    Submodules @@ -780,9 +779,9 @@

    Submodules + - - + @@ -793,7 +792,7 @@

    Submodules - + torch_tensorrt.ts — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,32 +305,32 @@
    - +
    @@ -344,20 +342,20 @@
    - - + +
    - +
    - +

    torch_tensorrt.ts

    Functions

    -torch_tensorrt.ts.compile(module: torch.jit._script.ScriptModule, inputs=[], device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_min_timing_iters=2, num_avg_timing_iters=1, workspace_size=0, calibrator=None, truncate_long_and_double=False, require_full_compilation=False, min_block_size=3, torch_executed_ops=[], torch_executed_modules=[]) → torch.jit._script.ScriptModule[source]
    +torch_tensorrt.ts.compile(module: torch.jit._script.ScriptModule, inputs=[], input_signature=None, device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_avg_timing_iters=1, workspace_size=0, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, calibrator=None, truncate_long_and_double=False, require_full_compilation=False, min_block_size=3, torch_executed_ops=[], torch_executed_modules=[]) → torch.jit._script.ScriptModule[source]

    Compile a TorchScript module for NVIDIA GPUs using TensorRT

    Takes a existing TorchScript module and a set of settings to configure the compiler and will convert methods to JIT Graphs which call equivalent TensorRT engines

    @@ -386,6 +384,21 @@

    Functions
    input_signature=([
+    torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+    torch_tensorrt.Input(
+        min_shape=(1, 224, 224, 3),
+        opt_shape=(1, 512, 512, 3),
+        max_shape=(1, 1024, 1024, 3),
+        dtype=torch.int32
+        format=torch.channel_last
+    ), # Dynamic input shape for input #2
+], torch.randn((1, 3, 224, 244))) # Use an example tensor and let torch_tensorrt infer settings for input #3
+
    +

    +

  • device (Union(torch_tensorrt.Device, torch.device, dict)) –

    Target device for TensorRT engines to run on

    device=torch_tensorrt.Device("dla:1", allow_gpu_fallback=True)
    @@ -397,9 +410,11 @@

    Functionstorch_tensorrt.EngineCapability) – Restrict kernel selection to safe gpu kernels or safe dla kernels

    • -

  • num_min_timing_iters (python:int) – Number of minimization timing iterations used to select kernels

  • num_avg_timing_iters (python:int) – Number of averaging timing iterations used to select kernels

  • workspace_size (python:int) – Maximum size of workspace given to TensorRT

    • +

  • dla_sram_size (python:int) – Fast software managed RAM used by DLA to communicate within a layer.

    • +

  • dla_local_dram_size (python:int) – Host RAM used by DLA to share intermediate tensor data across operations

    • +

  • dla_global_dram_size (python:int) – Host RAM used by DLA to store weights and metadata for execution

  • truncate_long_and_double (bool) – Truncate weights provided in int64 or double (float64) to int32 and float32

  • calibrator (Union(torch_tensorrt._C.IInt8Calibrator, tensorrt.IInt8Calibrator)) – Calibrator object which will provide data to the PTQ system for INT8 Calibration

  • require_full_compilation (bool) – Require modules to be compiled end to end or return an error as opposed to returning a hybrid graph where operations that cannot be run in TensorRT are run in PyTorch

    • @@ -419,7 +434,7 @@

    Functions
    -torch_tensorrt.ts.convert_method_to_trt_engine(module: torch.jit._script.ScriptModule, method_name: str, inputs=[], device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_min_timing_iters=2, num_avg_timing_iters=1, workspace_size=0, truncate_long_and_double=False, calibrator=None) → str[source]
    +torch_tensorrt.ts.convert_method_to_trt_engine(module: torch.jit._script.ScriptModule, method_name: str, inputs=[], device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_avg_timing_iters=1, workspace_size=0, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, truncate_long_and_double=False, calibrator=None) → str[source]

    Convert a TorchScript module method to a serialized TensorRT engine

    Converts a specified method of a module to a serialized TensorRT engine given a dictionary of conversion settings

    @@ -449,6 +464,21 @@

    Functions
    input_signature=([
+    torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+    torch_tensorrt.Input(
+        min_shape=(1, 224, 224, 3),
+        opt_shape=(1, 512, 512, 3),
+        max_shape=(1, 1024, 1024, 3),
+        dtype=torch.int32
+        format=torch.channel_last
+    ), # Dynamic input shape for input #2
+], torch.randn((1, 3, 224, 244))) # Use an example tensor and let torch_tensorrt infer settings for input #3
+
    +

    +

  • device (Union(torch_tensorrt.Device, torch.device, dict)) –

    Target device for TensorRT engines to run on

    device=torch_tensorrt.Device("dla:1", allow_gpu_fallback=True)
    @@ -460,9 +490,11 @@

    Functionstorch_tensorrt.EngineCapability) – Restrict kernel selection to safe gpu kernels or safe dla kernels

    • -

  • num_min_timing_iters (python:int) – Number of minimization timing iterations used to select kernels

  • num_avg_timing_iters (python:int) – Number of averaging timing iterations used to select kernels

  • workspace_size (python:int) – Maximum size of workspace given to TensorRT

    • +

  • dla_sram_size (python:int) – Fast software managed RAM used by DLA to communicate within a layer.

    • +

  • dla_local_dram_size (python:int) – Host RAM used by DLA to share intermediate tensor data across operations

    • +

  • dla_global_dram_size (python:int) – Host RAM used by DLA to store weights and metadata for execution

  • truncate_long_and_double (bool) – Truncate weights provided in int64 or double (float64) to int32 and float32

  • calibrator (Union(torch_tensorrt._C.IInt8Calibrator, tensorrt.IInt8Calibrator)) – Calibrator object which will provide data to the PTQ system for INT8 Calibration

    • @@ -535,7 +567,7 @@

    Functions
    -torch_tensorrt.ts.TensorRTCompileSpec(inputs=[], device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_min_timing_iters=2, num_avg_timing_iters=1, workspace_size=0, truncate_long_and_double=False, calibrator=None) → <torch._C.ScriptClass object at 0x7f9f791ec5b0>[source]
    +torch_tensorrt.ts.TensorRTCompileSpec(inputs=[], input_signature=None, device=None, disable_tf32=False, sparse_weights=False, enabled_precisions={}, refit=False, debug=False, capability=<EngineCapability.default: 0>, num_avg_timing_iters=1, workspace_size=0, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, truncate_long_and_double=False, calibrator=None) → <torch.ScriptClass object at 0x7f3b810fa3f0>[source]

    Utility to create a formated spec dictionary for using the PyTorch TensorRT backend

    Keyword Arguments
    @@ -568,7 +600,6 @@

    Functionstorch_tensorrt.EngineCapability) – Restrict kernel selection to safe gpu kernels or safe dla kernels

    -

  • num_min_timing_iters (python:int) – Number of minimization timing iterations used to select kernels

  • num_avg_timing_iters (python:int) – Number of averaging timing iterations used to select kernels

  • workspace_size (python:int) – Maximum size of workspace given to TensorRT

  • truncate_long_and_double (bool) – Truncate weights provided in int64 or double (float64) to int32 and float32

    • @@ -583,25 +614,25 @@

    Functions - - - - + + + + - +

    + - - +
    - +

    @@ -609,11 +640,11 @@

    FunctionsSphinx using a theme provided by Read the Docs.

    - + @@ -636,12 +667,12 @@

    Functions @@ -650,9 +681,9 @@

    Functions + - - + @@ -663,7 +694,7 @@

    Functions - + - + Search — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,11 +33,11 @@ - + + - - + @@ -177,9 +177,9 @@ + - - +
    Table of Contents @@ -190,21 +190,21 @@
    @@ -285,7 +283,7 @@
    - + @@ -304,28 +302,28 @@
    - +
    @@ -337,13 +335,13 @@
    - - + +
    - +
    - + - +
    - +
    - +
    @@ -390,19 +388,19 @@
    - +
    + + - - - + @@ -413,9 +411,9 @@ + - - + @@ -428,7 +426,7 @@ }); - + diff --git a/docs/searchindex.js b/docs/searchindex.js index 9d9da03a0c..f481d51c86 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["_cpp_api/classtorch__tensorrt_1_1DataType","_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType","_cpp_api/classtorch__tensorrt_1_1TensorFormat","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator","_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502","_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268","_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e","_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827","_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b","_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da","_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59","_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883","_cpp_api/dir_cpp","_cpp_api/dir_cpp_include","_cpp_api/dir_cpp_include_torch_tensorrt","_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558","_cpp_api/enum_torch__tensorrt_8h_1a3fbe5d72e4fc624dbd038853079620eb","_cpp_api/file_cpp_include_torch_tensorrt_logging.h","_cpp_api/file_cpp_include_torch_tensorrt_macros.h","_cpp_api/file_cpp_include_torch_tensorrt_ptq.h","_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h","_cpp_api/function_logging_8h_1a0593f776f469c20469e2f729fc7861a3","_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650","_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a","_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2","_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8","_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5","_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc","_cpp_api/function_ptq_8h_1a83ff2be7e0b80bc7434de415861dc039","_cpp_api/function_ptq_8h_1a9835f6e605dce1abf442a55b64d6dffa","_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797","_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9","_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef","_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528","_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384","_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1","_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2","_cpp_api/namespace_torch_tensorrt","_cpp_api/namespace_torch_tensorrt__logging","_cpp_api/namespace_torch_tensorrt__ptq","_cpp_api/namespace_torch_tensorrt__torchscript","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h","_cpp_api/structtorch__tensorrt_1_1Device","_cpp_api/structtorch__tensorrt_1_1Input","_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec","_cpp_api/torch_tensort_cpp","_cpp_api/unabridged_orphan","_notebooks/CitriNet-example","_notebooks/EfficientNet-example","_notebooks/Hugging-Face-BERT","_notebooks/Resnet50-example","_notebooks/dynamic-shapes","_notebooks/lenet-getting-started","_notebooks/ssd-object-detection-demo","_notebooks/vgg-qat","contributors/conversion","contributors/lowering","contributors/partitioning","contributors/phases","contributors/runtime","contributors/system_overview","contributors/useful_links","contributors/writing_converters","index","indices/supported_ops","py_api/logging","py_api/ptq","py_api/torch_tensorrt","py_api/ts","src/pytorch-sphinx-theme/docs/changelog","src/pytorch-sphinx-theme/docs/configuring","src/pytorch-sphinx-theme/docs/demo/api","src/pytorch-sphinx-theme/docs/demo/demo","src/pytorch-sphinx-theme/docs/demo/lists_tables","src/pytorch-sphinx-theme/docs/demo/long","src/pytorch-sphinx-theme/docs/demo/structure","src/pytorch-sphinx-theme/docs/index","src/pytorch-sphinx-theme/docs/installing","tutorials/creating_torchscript_module_in_python","tutorials/getting_started_with_cpp_api","tutorials/getting_started_with_python_api","tutorials/installation","tutorials/ptq","tutorials/runtime","tutorials/serving_torch_tensorrt_with_triton","tutorials/torchtrtc","tutorials/use_from_pytorch","tutorials/using_dla"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1,nbsphinx:4,sphinx:56},filenames:["_cpp_api/classtorch__tensorrt_1_1DataType.rst","_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.rst","_cpp_api/classtorch__tensorrt_1_1TensorFormat.rst","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.rst","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.rst","_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.rst","_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.rst","_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.rst","_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.rst","_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.rst","_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.rst","_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.rst","_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.rst","_cpp_api/dir_cpp.rst","_cpp_api/dir_cpp_include.rst","_cpp_api/dir_cpp_include_torch_tensorrt.rst","_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.rst","_cpp_api/enum_torch__tensorrt_8h_1a3fbe5d72e4fc624dbd038853079620eb.rst","_cpp_api/file_cpp_include_torch_tensorrt_logging.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst","_cpp_api/function_logging_8h_1a0593f776f469c20469e2f729fc7861a3.rst","_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.rst","_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.rst","_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.rst","_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.rst","_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.rst","_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.rst","_cpp_api/function_ptq_8h_1a83ff2be7e0b80bc7434de415861dc039.rst","_cpp_api/function_ptq_8h_1a9835f6e605dce1abf442a55b64d6dffa.rst","_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.rst","_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.rst","_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.rst","_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.rst","_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.rst","_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.rst","_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.rst","_cpp_api/namespace_torch_tensorrt.rst","_cpp_api/namespace_torch_tensorrt__logging.rst","_cpp_api/namespace_torch_tensorrt__ptq.rst","_cpp_api/namespace_torch_tensorrt__torchscript.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst","_cpp_api/structtorch__tensorrt_1_1Device.rst","_cpp_api/structtorch__tensorrt_1_1Input.rst","_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.rst","_cpp_api/torch_tensort_cpp.rst","_cpp_api/unabridged_orphan.rst","_notebooks/CitriNet-example.ipynb","_notebooks/EfficientNet-example.ipynb","_notebooks/Hugging-Face-BERT.ipynb","_notebooks/Resnet50-example.ipynb","_notebooks/dynamic-shapes.ipynb","_notebooks/lenet-getting-started.ipynb","_notebooks/ssd-object-detection-demo.ipynb","_notebooks/vgg-qat.ipynb","contributors/conversion.rst","contributors/lowering.rst","contributors/partitioning.rst","contributors/phases.rst","contributors/runtime.rst","contributors/system_overview.rst","contributors/useful_links.rst","contributors/writing_converters.rst","index.rst","indices/supported_ops.rst","py_api/logging.rst","py_api/ptq.rst","py_api/torch_tensorrt.rst","py_api/ts.rst","src/pytorch-sphinx-theme/docs/changelog.rst","src/pytorch-sphinx-theme/docs/configuring.rst","src/pytorch-sphinx-theme/docs/demo/api.rst","src/pytorch-sphinx-theme/docs/demo/demo.rst","src/pytorch-sphinx-theme/docs/demo/lists_tables.rst","src/pytorch-sphinx-theme/docs/demo/long.rst","src/pytorch-sphinx-theme/docs/demo/structure.rst","src/pytorch-sphinx-theme/docs/index.rst","src/pytorch-sphinx-theme/docs/installing.rst","tutorials/creating_torchscript_module_in_python.rst","tutorials/getting_started_with_cpp_api.rst","tutorials/getting_started_with_python_api.rst","tutorials/installation.rst","tutorials/ptq.rst","tutorials/runtime.rst","tutorials/serving_torch_tensorrt_with_triton.rst","tutorials/torchtrtc.rst","tutorials/use_from_pytorch.rst","tutorials/using_dla.rst"],objects:{"":[[5,0,1,"c.STR","STR"],[9,0,1,"c.TORCHTRT_API","TORCHTRT_API"],[11,0,1,"c.TORCHTRT_HIDDEN","TORCHTRT_HIDDEN"],[7,0,1,"c.TORCH_TENSORRT_MAJOR_VERSION","TORCH_TENSORRT_MAJOR_VERSION"],[8,0,1,"c.TORCH_TENSORRT_MINOR_VERSION","TORCH_TENSORRT_MINOR_VERSION"],[6,0,1,"c.TORCH_TENSORRT_PATCH_VERSION","TORCH_TENSORRT_PATCH_VERSION"],[12,0,1,"c.TORCH_TENSORRT_VERSION","TORCH_TENSORRT_VERSION"],[10,0,1,"c.XSTR","XSTR"],[0,1,1,"_CPPv4N14torch_tensorrt8DataTypeE","torch_tensorrt::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value","torch_tensorrt::DataType::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE","torch_tensorrt::DataType::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEv","torch_tensorrt::DataType::DataType"],[0,3,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value","torch_tensorrt::DataType::DataType::t"],[0,3,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE","torch_tensorrt::DataType::DataType::t"],[0,4,1,"_CPPv4N14torch_tensorrt8DataType5ValueE","torch_tensorrt::DataType::Value"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kBoolE","torch_tensorrt::DataType::Value::kBool"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kCharE","torch_tensorrt::DataType::Value::kChar"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value6kFloatE","torch_tensorrt::DataType::Value::kFloat"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kHalfE","torch_tensorrt::DataType::Value::kHalf"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value4kIntE","torch_tensorrt::DataType::Value::kInt"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE","torch_tensorrt::DataType::Value::kUnknown"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kBoolE","torch_tensorrt::DataType::kBool"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kCharE","torch_tensorrt::DataType::kChar"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value6kFloatE","torch_tensorrt::DataType::kFloat"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kHalfE","torch_tensorrt::DataType::kHalf"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value4kIntE","torch_tensorrt::DataType::kInt"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE","torch_tensorrt::DataType::kUnknown"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypecv5ValueEv","torch_tensorrt::DataType::operator Value"],[0,2,1,"_CPPv4N14torch_tensorrt8DataTypecvbEv","torch_tensorrt::DataType::operator bool"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeneE8DataType","torch_tensorrt::DataType::operator!="],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE","torch_tensorrt::DataType::operator!="],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeneE8DataType","torch_tensorrt::DataType::operator!=::other"],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE","torch_tensorrt::DataType::operator!=::other"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType","torch_tensorrt::DataType::operator=="],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE","torch_tensorrt::DataType::operator=="],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType","torch_tensorrt::DataType::operator==::other"],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE","torch_tensorrt::DataType::operator==::other"],[46,1,1,"_CPPv4N14torch_tensorrt6DeviceE","torch_tensorrt::Device"],[46,2,1,"_CPPv4N14torch_tensorrt6Device6DeviceEv","torch_tensorrt::Device::Device"],[1,1,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypeE","torch_tensorrt::Device::DeviceType"],[46,1,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypeE","torch_tensorrt::Device::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv","torch_tensorrt::Device::DeviceType::DeviceType"],[1,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType::t"],[1,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType::t"],[46,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType::t"],[46,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType::t"],[1,4,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE","torch_tensorrt::Device::DeviceType::Value"],[46,4,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE","torch_tensorrt::Device::DeviceType::Value"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::Value::kDLA"],[46,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::Value::kDLA"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::Value::kGPU"],[46,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::Value::kGPU"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::kDLA"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::kGPU"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv","torch_tensorrt::Device::DeviceType::operator Value"],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv","torch_tensorrt::Device::DeviceType::operator Value"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv","torch_tensorrt::Device::DeviceType::operator bool"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv","torch_tensorrt::Device::DeviceType::operator bool"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!="],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!="],[1,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!=::other"],[46,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!=::other"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator=="],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator=="],[1,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator==::other"],[46,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator==::other"],[46,6,1,"_CPPv4N14torch_tensorrt6Device18allow_gpu_fallbackE","torch_tensorrt::Device::allow_gpu_fallback"],[46,6,1,"_CPPv4N14torch_tensorrt6Device11device_typeE","torch_tensorrt::Device::device_type"],[46,6,1,"_CPPv4N14torch_tensorrt6Device8dla_coreE","torch_tensorrt::Device::dla_core"],[46,6,1,"_CPPv4N14torch_tensorrt6Device6gpu_idE","torch_tensorrt::Device::gpu_id"],[17,4,1,"_CPPv4N14torch_tensorrt16EngineCapabilityE","torch_tensorrt::EngineCapability"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE","torch_tensorrt::EngineCapability::kDLA_STANDALONE"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE","torch_tensorrt::EngineCapability::kSAFETY"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE","torch_tensorrt::EngineCapability::kSTANDARD"],[47,1,1,"_CPPv4N14torch_tensorrt5InputE","torch_tensorrt::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[47,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::shape"],[47,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE","torch_tensorrt::Input::Input::tensor"],[47,6,1,"_CPPv4N14torch_tensorrt5Input5dtypeE","torch_tensorrt::Input::dtype"],[47,6,1,"_CPPv4N14torch_tensorrt5Input6formatE","torch_tensorrt::Input::format"],[47,6,1,"_CPPv4N14torch_tensorrt5Input9max_shapeE","torch_tensorrt::Input::max_shape"],[47,6,1,"_CPPv4N14torch_tensorrt5Input9min_shapeE","torch_tensorrt::Input::min_shape"],[47,6,1,"_CPPv4N14torch_tensorrt5Input9opt_shapeE","torch_tensorrt::Input::opt_shape"],[47,6,1,"_CPPv4N14torch_tensorrt5Input5shapeE","torch_tensorrt::Input::shape"],[2,1,1,"_CPPv4N14torch_tensorrt12TensorFormatE","torch_tensorrt::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value","torch_tensorrt::TensorFormat::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE","torch_tensorrt::TensorFormat::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEv","torch_tensorrt::TensorFormat::TensorFormat"],[2,3,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value","torch_tensorrt::TensorFormat::TensorFormat::t"],[2,3,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE","torch_tensorrt::TensorFormat::TensorFormat::t"],[2,4,1,"_CPPv4N14torch_tensorrt12TensorFormat5ValueE","torch_tensorrt::TensorFormat::Value"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE","torch_tensorrt::TensorFormat::Value::kChannelsLast"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE","torch_tensorrt::TensorFormat::Value::kContiguous"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE","torch_tensorrt::TensorFormat::Value::kUnknown"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE","torch_tensorrt::TensorFormat::kChannelsLast"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE","torch_tensorrt::TensorFormat::kContiguous"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE","torch_tensorrt::TensorFormat::kUnknown"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatcv5ValueEv","torch_tensorrt::TensorFormat::operator Value"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormatcvbEv","torch_tensorrt::TensorFormat::operator bool"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat","torch_tensorrt::TensorFormat::operator!="],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator!="],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat","torch_tensorrt::TensorFormat::operator!=::other"],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator!=::other"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat","torch_tensorrt::TensorFormat::operator=="],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator=="],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat","torch_tensorrt::TensorFormat::operator==::other"],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator==::other"],[36,2,1,"_CPPv4N14torch_tensorrt15dump_build_infoEv","torch_tensorrt::dump_build_info"],[34,2,1,"_CPPv4N14torch_tensorrt14get_build_infoEv","torch_tensorrt::get_build_info"],[16,4,1,"_CPPv4N14torch_tensorrt7logging5LevelE","torch_tensorrt::logging::Level"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE","torch_tensorrt::logging::Level::kDEBUG"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kERRORE","torch_tensorrt::logging::Level::kERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE","torch_tensorrt::logging::Level::kGRAPH"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level5kINFOE","torch_tensorrt::logging::Level::kINFO"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE","torch_tensorrt::logging::Level::kINTERNAL_ERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE","torch_tensorrt::logging::Level::kWARNING"],[24,2,1,"_CPPv4N14torch_tensorrt7logging24get_is_colored_output_onEv","torch_tensorrt::logging::get_is_colored_output_on"],[22,2,1,"_CPPv4N14torch_tensorrt7logging18get_logging_prefixEv","torch_tensorrt::logging::get_logging_prefix"],[23,2,1,"_CPPv4N14torch_tensorrt7logging24get_reportable_log_levelEv","torch_tensorrt::logging::get_reportable_log_level"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE","torch_tensorrt::logging::kDEBUG"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kERRORE","torch_tensorrt::logging::kERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE","torch_tensorrt::logging::kGRAPH"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level5kINFOE","torch_tensorrt::logging::kINFO"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE","torch_tensorrt::logging::kINTERNAL_ERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE","torch_tensorrt::logging::kWARNING"],[26,2,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log"],[26,3,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log::lvl"],[26,3,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log::msg"],[27,2,1,"_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb","torch_tensorrt::logging::set_is_colored_output_on"],[27,3,1,"_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb","torch_tensorrt::logging::set_is_colored_output_on::colored_output_on"],[28,2,1,"_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE","torch_tensorrt::logging::set_logging_prefix"],[28,3,1,"_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE","torch_tensorrt::logging::set_logging_prefix::prefix"],[25,2,1,"_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level","torch_tensorrt::logging::set_reportable_log_level"],[25,3,1,"_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level","torch_tensorrt::logging::set_reportable_log_level::lvl"],[3,1,1,"_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE","torch_tensorrt::ptq::Int8CacheCalibrator"],[3,7,1,"_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE","torch_tensorrt::ptq::Int8CacheCalibrator::Algorithm"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE","torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE","torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator::cache_file_path"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::bindings"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::names"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::nbBindings"],[3,2,1,"_CPPv4NK14torch_tensorrt3ptq19Int8CacheCalibrator12getBatchSizeEv","torch_tensorrt::ptq::Int8CacheCalibrator::getBatchSize"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibratorcvPN8nvinfer115IInt8CalibratorEEv","torch_tensorrt::ptq::Int8CacheCalibrator::operator nvinfer1::IInt8Calibrator*"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::readCalibrationCache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::readCalibrationCache::length"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache::cache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache::length"],[4,1,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator"],[4,7,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator::Algorithm"],[4,7,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator::DataLoaderUniquePtr"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::cache_file_path"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::dataloader"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::use_cache"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::bindings"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::names"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::nbBindings"],[4,2,1,"_CPPv4NK14torch_tensorrt3ptq14Int8Calibrator12getBatchSizeEv","torch_tensorrt::ptq::Int8Calibrator::getBatchSize"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8CalibratorcvPN8nvinfer115IInt8CalibratorEEv","torch_tensorrt::ptq::Int8Calibrator::operator nvinfer1::IInt8Calibrator*"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8Calibrator::readCalibrationCache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8Calibrator::readCalibrationCache::length"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache::cache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache::length"],[30,2,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator"],[30,7,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator::Algorithm"],[30,3,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator::cache_file_path"],[29,2,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator"],[29,7,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::Algorithm"],[29,7,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::DataLoader"],[29,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::cache_file_path"],[29,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::dataloader"],[29,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::use_cache"],[35,2,1,"_CPPv4N14torch_tensorrt10set_deviceEKi","torch_tensorrt::set_device"],[35,3,1,"_CPPv4N14torch_tensorrt10set_deviceEKi","torch_tensorrt::set_device::gpu_id"],[48,1,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpecE","torch_tensorrt::torchscript::CompileSpec"],[48,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[48,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[48,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[48,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"],[48,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"],[48,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::inputs"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec10capabilityE","torch_tensorrt::torchscript::CompileSpec::capability"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec5debugE","torch_tensorrt::torchscript::CompileSpec::debug"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec6deviceE","torch_tensorrt::torchscript::CompileSpec::device"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec12disable_tf32E","torch_tensorrt::torchscript::CompileSpec::disable_tf32"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec18enabled_precisionsE","torch_tensorrt::torchscript::CompileSpec::enabled_precisions"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec6inputsE","torch_tensorrt::torchscript::CompileSpec::inputs"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14min_block_sizeE","torch_tensorrt::torchscript::CompileSpec::min_block_size"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec20num_avg_timing_itersE","torch_tensorrt::torchscript::CompileSpec::num_avg_timing_iters"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec20num_min_timing_itersE","torch_tensorrt::torchscript::CompileSpec::num_min_timing_iters"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14ptq_calibratorE","torch_tensorrt::torchscript::CompileSpec::ptq_calibrator"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec5refitE","torch_tensorrt::torchscript::CompileSpec::refit"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec24require_full_compilationE","torch_tensorrt::torchscript::CompileSpec::require_full_compilation"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14sparse_weightsE","torch_tensorrt::torchscript::CompileSpec::sparse_weights"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec22torch_executed_modulesE","torch_tensorrt::torchscript::CompileSpec::torch_executed_modules"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec18torch_executed_opsE","torch_tensorrt::torchscript::CompileSpec::torch_executed_ops"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec24truncate_long_and_doubleE","torch_tensorrt::torchscript::CompileSpec::truncate_long_and_double"],[48,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14workspace_sizeE","torch_tensorrt::torchscript::CompileSpec::workspace_size"],[31,2,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support"],[31,3,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support::method_name"],[31,3,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support::module"],[32,2,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile"],[32,3,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile::info"],[32,3,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile::module"],[37,2,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::info"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::method_name"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::module"],[33,2,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module"],[33,3,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module::device"],[33,3,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module::engine"],[71,8,0,"-","torch_tensorrt"]],"torch_tensorrt.Device":[[71,10,1,"","__init__"],[71,11,1,"","allow_gpu_fallback"],[71,11,1,"","device_type"],[71,11,1,"","dla_core"],[71,11,1,"","gpu_id"]],"torch_tensorrt.Input":[[71,10,1,"","__init__"],[71,11,1,"","dtype"],[71,11,1,"","format"],[71,11,1,"","shape"],[71,11,1,"","shape_mode"]],"torch_tensorrt.logging":[[69,9,1,"","Level"],[69,9,1,"","debug"],[69,9,1,"","errors"],[69,12,1,"","get_is_colored_output_on"],[69,12,1,"","get_logging_prefix"],[69,12,1,"","get_reportable_log_level"],[69,9,1,"","graphs"],[69,9,1,"","info"],[69,9,1,"","internal_errors"],[69,12,1,"","log"],[69,12,1,"","set_is_colored_output_on"],[69,12,1,"","set_logging_prefix"],[69,12,1,"","set_reportable_log_level"],[69,9,1,"","warnings"]],"torch_tensorrt.logging.Level":[[69,11,1,"","Debug"],[69,11,1,"","Error"],[69,11,1,"","Graph"],[69,11,1,"","Info"],[69,11,1,"","InternalError"],[69,11,1,"","Warning"]],"torch_tensorrt.ptq":[[70,9,1,"id1","CacheCalibrator"],[70,9,1,"id2","CalibrationAlgo"],[70,9,1,"id0","DataLoaderCalibrator"],[70,12,1,"","get_batch"],[70,12,1,"","get_batch_size"],[70,12,1,"","get_cache_mode_batch"],[70,12,1,"","read_calibration_cache"],[70,12,1,"","write_calibration_cache"]],"torch_tensorrt.ptq.CacheCalibrator":[[70,10,1,"","__init__"]],"torch_tensorrt.ptq.CalibrationAlgo":[[70,11,1,"","ENTROPY_CALIBRATION"],[70,11,1,"","ENTROPY_CALIBRATION_2"],[70,11,1,"","LEGACY_CALIBRATION"],[70,11,1,"","MINMAX_CALIBRATION"]],"torch_tensorrt.ptq.DataLoaderCalibrator":[[70,10,1,"","__init__"]],"torch_tensorrt.ts":[[72,12,1,"","TensorRTCompileSpec"],[72,12,1,"","check_method_op_support"],[72,12,1,"","compile"],[72,12,1,"","convert_method_to_trt_engine"],[72,12,1,"","embed_engine_in_new_module"]],torch_tensorrt:[[71,9,1,"","Device"],[71,9,1,"","DeviceType"],[71,9,1,"","EngineCapability"],[71,9,1,"","Input"],[71,9,1,"","TensorFormat"],[71,12,1,"","compile"],[71,12,1,"","convert_method_to_trt_engine"],[71,9,1,"","dtype"],[71,12,1,"","dump_build_info"],[71,12,1,"","get_build_info"],[69,8,0,"-","logging"],[70,8,0,"-","ptq"],[71,12,1,"","set_device"],[72,8,0,"-","ts"]]},objnames:{"0":["c","macro","C macro"],"1":["cpp","class","C++ class"],"10":["py","method","Python method"],"11":["py","attribute","Python attribute"],"12":["py","function","Python function"],"2":["cpp","function","C++ function"],"3":["cpp","functionParam","C++ function parameter"],"4":["cpp","enum","C++ enum"],"5":["cpp","enumerator","C++ enumerator"],"6":["cpp","member","C++ member"],"7":["cpp","templateParam","C++ template parameter"],"8":["py","module","Python module"],"9":["py","class","Python class"]},objtypes:{"0":"c:macro","1":"cpp:class","10":"py:method","11":"py:attribute","12":"py:function","2":"cpp:function","3":"cpp:functionParam","4":"cpp:enum","5":"cpp:enumerator","6":"cpp:member","7":"cpp:templateParam","8":"py:module","9":"py:class"},terms:{"0":[33,43,44,45,48,51,52,53,54,55,56,57,58,64,66,68,69,70,71,72,73,75,76,83,85,86,88,89,90,91],"00":[51,52,54,55,56,57,58],"0000":77,"00000000":[51,52,54,55,56],"000000037777":57,"000000252219":57,"000000397133":57,"000007":53,"000014":51,"000015":53,"000059":53,"000106":51,"000116":51,"000368":51,"000545":51,"000820":51,"000973":51,"001256":51,"001260":51,"001270":51,"001351":51,"0018":58,"002":54,"002251":53,"002259":53,"0023":58,"002305":53,"0026":58,"003287":53,"003289":53,"003317":53,"003462":51,"003774":51,"004":52,"004128":51,"004205":53,"004206":53,"004256":53,"004825":51,"005":[54,55],"006":[52,55],"006661":51,"006677":53,"006693":53,"006733":51,"006846":51,"006943":53,"0070":58,"008":58,"008071":51,"008453":51,"0087":58,"009802":51,"009803":51,"009836":51,"00f1b6db":[52,54,55],"01":[52,54,55,56,57,58,68,77,83],"0106":58,"010961":51,"011388":51,"013":58,"0151":58,"016114":51,"0163":58,"0169":58,"018642":51,"018643":51,"018670":51,"02":[52,54,55,58],"0208":83,"020804":51,"021143":51,"0220":58,"024492":51,"025":58,"025000":58,"0263":58,"028":58,"0296":58,"03":[51,77],"03291":58,"033488":51,"033572":51,"03466":58,"035722":51,"0358":83,"0383":83,"04":[51,52,57,58,83,85,88],"0435":83,"04609":58,"0464":83,"04743":58,"04807":58,"0491":58,"0493":58,"04it":58,"05":[51,52,53,54,55,57,58],"050000":58,"0505":58,"05080":58,"0530":83,"05311":58,"05374":58,"057":58,"058047":51,"058053":51,"058375":51,"05945":58,"06":[51,52,57],"0622":58,"063":58,"06340":58,"06567":58,"0676ba61":[54,57],"0678":83,"069":58,"07":[52,54,55],"071":58,"071428":51,"072057":51,"07266":58,"076796":51,"08":[52,54,55],"0805":83,"0818":83,"08331":58,"08555":58,"086":58,"09":[52,54,55,56],"0932":83,"096":58,"0a0":[51,52,76],"0a3":51,"0c106ec84f199a0fbcf1199010166986da732f9b0907768c9ac5ea5b120772db":85,"0f":57,"0mib":[52,54,55,56],"0rc1":51,"0s":54,"0x":53,"0x7f9f791ec5b0":72,"1":[3,4,33,43,44,45,47,48,51,52,53,54,55,56,57,58,60,61,63,66,68,69,70,71,72,73,74,76,77,80,82,83,84,85,86,89,90,91],"10":[48,51,52,53,54,55,56,57,58,72,80,82,83,85,86,88],"100":[52,54,55,56,57,58],"1000":[52,54,55,56,57,58,88],"10000":[52,54,55],"100000":58,"10018":58,"10070":58,"101":53,"101168":58,"1012":60,"1013":60,"10130":58,"102":51,"102248":58,"1024":[51,52,54,55,56,57,58,71,72,89],"10240mib":56,"10362":52,"104":[52,54,55],"1045":83,"105":58,"1056":83,"1063":83,"1065":51,"1069":51,"107":[52,54,55],"107194":58,"10732":58,"107625":58,"109":83,"10990":58,"10b0":51,"11":[51,52,53,54,55,56,57,58,60,76,80,83,85,88],"110":[57,58],"11299":58,"112mib":51,"11499":58,"115":56,"115269":58,"115740":58,"11594":58,"117":[52,54,55],"117969":58,"118358":58,"11879":58,"11888":58,"119":82,"1190":51,"119708":51,"11k":[52,54,55],"11w":52,"12":[51,52,53,54,55,56,57,58,60,76,80,82,83,88],"120":[56,58,82,83],"120097":51,"1201":51,"121":[54,56],"1216":53,"121618":51,"122":56,"12288mib":51,"123":[57,77],"12345":51,"126":58,"126382":58,"126834":58,"127":[52,58],"128":[51,52,53,54,55,56,57,58],"128674":58,"129":82,"129518":58,"12k":54,"13":[51,53,54,55,56,57,58,76,80,85],"130":51,"133":52,"13388":58,"135453":58,"135936":58,"136":88,"137":[51,82],"137858":58,"138":82,"138366":58,"139704147265344":58,"13x":52,"14":[51,52,53,54,55,56,57,58,80,88],"1409":86,"141":58,"143":51,"145":58,"145539":58,"146":51,"146053":58,"147871":58,"148353":58,"1488":51,"149":51,"14x":53,"15":[51,53,54,55,56,57,58,76,80],"1500":58,"1502":83,"1516":51,"1531":58,"1535566590":[52,54,55],"1538":51,"154252":58,"154685":58,"1549":[55,83],"1552":55,"1556":86,"1560":55,"1563":58,"156558":58,"1566":55,"1568":55,"157159":58,"1572":55,"1574":55,"1575":55,"1598":55,"15w":54,"15x":53,"16":[51,53,54,55,56,57,58,71,80,82,83,84],"16000":51,"163197":58,"163676":58,"164":[52,54,55],"165":57,"165549":58,"165991":58,"166":57,"167":57,"1691":83,"17":[51,52,53,54,55,56,57,58,80],"173":58,"173305":58,"173926":58,"176034":58,"176697":58,"1771":53,"1776":53,"1777":[51,58],"179":57,"1792":51,"18":[51,52,53,54,55,56,57,58,80,83,85],"182843":58,"183426":58,"185377":58,"185962":58,"188":58,"19":[51,53,54,57,58,77,80],"1906":58,"191966":58,"192424":58,"194325":58,"194817":58,"1971":58,"198":51,"1994":[58,86],"1d":60,"1e":[54,55,58,89],"1f":[51,53],"1rc0":51,"1ubuntu0":51,"1x1":57,"2":[33,43,45,48,51,52,53,54,55,56,57,58,61,66,68,69,70,71,72,74,76,77,80,82,83,85,86,90],"20":[51,52,53,54,55,57,58,80],"200":[52,54,55,56,58],"2000000000":[51,53],"2002":58,"2009":86,"200w":55,"201":[52,54,55],"2010":[58,86],"2012":77,"2014":86,"2017":[51,53,57],"2018":[52,53,54,55],"2019":[51,52,53,54,56,57],"201988":58,"202":[52,54,55],"2020":[57,58,67,83],"2021":[51,53],"2022":[51,52,53,54,55,56,57,58],"2023":[58,86],"202665":58,"204763":58,"2048":[54,55],"205461":58,"20w":56,"21":[51,52,53,54,55,56,57,58],"211393":58,"211987":58,"213899":58,"214450":58,"215434":51,"215446":51,"215806":51,"216":52,"217":[54,55],"218":51,"22":[51,52,53,54,55,56,57,58,88],"220892":58,"221533":58,"222":54,"223":[54,55],"223519":58,"224":[52,54,55,61,71,72,88],"224037":58,"225":[52,54,55,88],"227":[52,54,55],"227739155292511":52,"229":[52,54,55,88],"23":[48,51,52,54,55,58,60,72,77],"2305":58,"23344755172729492":52,"233809":58,"234":58,"234375":88,"234434":58,"235":51,"237":58,"238":[55,58],"238212":58,"239042":58,"24":[51,54,56,57,58,60],"241022":58,"24112":[52,54,55],"241654":58,"242":51,"243":[54,56],"244":[71,72],"245":57,"2453mib":51,"24576mib":[52,54],"246":52,"2462mib":51,"246kb":52,"247820":58,"248":60,"248445":58,"249":60,"24k":[52,54,55],"25":[51,54,55,58,83],"250366":58,"250959":58,"250w":51,"254":58,"256":[52,54,55,58,88],"257248":58,"257854":58,"258":76,"259968":58,"26":[51,53,54,55,57],"2606":[52,54,55],"260660":58,"265":51,"268160":58,"26w":51,"27":[51,52,53,56,58,83],"272":51,"28":[51,52,55,83,86,91],"280":58,"2802":83,"282":51,"2822":76,"285":58,"287":76,"288":[51,58],"28c":52,"29":[51,52,55,58,83],"291":58,"29c":54,"2_20200626":85,"2c3":77,"2c365_subsampl":[52,54,55],"2c916ef":51,"2f":[52,54,55,56,57,58],"2s":54,"2x":54,"3":[45,48,51,52,53,54,55,56,57,58,60,61,63,68,69,70,71,72,76,77,80,82,83,85,86,89,90,91],"30":[52,54,55,57,58],"300":[56,57,58,89,90],"300x300":57,"302":58,"309":58,"3090":[52,54],"31":[51,54,55,56,57,83],"311":58,"314":58,"315":51,"32":[51,52,53,55,56,57,58,71,82,83,84,86,89,91],"320":86,"3207":58,"320w":56,"321":52,"329273":58,"32bit":89,"32x32":54,"33":[52,54,55,56,57,83],"330212":58,"332529":58,"333365":58,"3393":52,"339547":58,"34":[52,54,55,56,57,58],"340248":58,"342257":58,"342890":58,"345":58,"346":83,"349":51,"35":[52,54,57,83],"350619":58,"350w":[52,54],"351372":58,"352":[52,54,55],"353470":58,"35363":[52,54,55],"353k":[52,54,55],"354121":58,"3550":58,"35k":[52,54,55],"35x":52,"36":[51,52,55,83],"360090":58,"360806":58,"361413":[52,54,55],"362803":58,"3631":58,"363274":58,"366":54,"366kb":54,"3677":60,"37":[51,52,54,55,58,83],"370369":58,"371057":58,"373071":58,"373766":58,"376":52,"3763":58,"379890":58,"38":[51,54,55,57,82],"380538":58,"382532":58,"383128":58,"385":58,"3877":58,"389077":58,"389760":58,"39":[51,52,53,54,55,56,57,58,82],"3909":51,"391815":58,"392399":58,"394":58,"39485082030296326":54,"395":58,"3987298309803009":52,"399809":58,"39c":51,"39mib":51,"3f":58,"3x3":58,"4":[51,52,53,54,55,56,57,58,63,68,69,74,76,77,80,83,85],"40":[52,54,55,56,57,58],"400":[56,58],"400472":58,"402399":58,"402939":58,"406":[52,54,55,88],"408818":58,"409424":58,"4096":58,"40mb":54,"41":[51,54,55,56],"411513":58,"4116":55,"412097":58,"4122":55,"4123":55,"4142":55,"4156":55,"4161":51,"4166":55,"4170":55,"4172":55,"4176":55,"4178":55,"418537":58,"419128":58,"42":[51,55,56,57,58],"421343":58,"421946":58,"429":51,"429382":58,"429688":88,"42c":56,"42w":51,"43":[51,56,57,58],"430156":58,"432259":58,"433079":58,"4352":58,"439":58,"439297":58,"44":[51,57,58],"440027":58,"442":[52,54,55,58],"442149":58,"442826":58,"442k":[52,54,55],"443":[52,54,55],"4465":[58,86],"449377":58,"449968":58,"45":[51,52,57],"452122":58,"452718":[52,54,55],"452754":58,"456":[52,54,55,88],"45675724744796753":55,"4584":52,"459":58,"46":[51,52,57,58],"462532":58,"463295":58,"466963":58,"467725":58,"468750":88,"469692":58,"47":51,"470":[55,58],"4700":[52,54,55],"470336":58,"4726":58,"474":52,"476204":58,"4767":55,"476738":58,"47681mib":55,"478809":58,"479375":58,"48":[51,54,55],"481":54,"4822":[58,86],"484":58,"485":[52,54,55,88],"485666":58,"486219":58,"488416":58,"488986":58,"489":55,"49":[51,53,57],"4914":[58,86],"4935":55,"49785590171813965":54,"49788108468055725":55,"4980":55,"499":58,"4fef":[52,54,55],"4mib":51,"4s":52,"4x":51,"5":[51,52,53,54,55,56,57,58,63,64,69,71,76,77,80,82,83,85,88,89],"50":[51,52,53,55,56,57,58],"500":[56,58],"5002":55,"5005":55,"5014":55,"5016":55,"5018":55,"5020":55,"5024":55,"5026":55,"5027":55,"5033":55,"504":58,"5052":55,"5067":55,"5088":55,"5091":55,"5094":55,"5096":55,"510":[51,52,54,56],"5100":55,"511":58,"5110":55,"5115":55,"5117":58,"5118":55,"512":[51,54,55,58,71,72,89],"512364":58,"513354":58,"514046":58,"514638":58,"515270":58,"5153":55,"515859":58,"516441":58,"517009":58,"5172":58,"517600":58,"518167":58,"518752":58,"519333":58,"5197":55,"519911":58,"51c":55,"52":[52,54,55,58],"5202":55,"520473":58,"5207":55,"521038":58,"5215":55,"521596":58,"522170":58,"522742":58,"5231":55,"523360":58,"523438":88,"523957":58,"5242":55,"524581":58,"525059":58,"525366":58,"525675":58,"525962":58,"526257":58,"526566":58,"526885":58,"527188":58,"527489":58,"527792":58,"528097":58,"528387":58,"528834":58,"529163":58,"53":[51,54,57,77],"5320":58,"532748":58,"533468":58,"5335":58,"534033":58,"534684":58,"535320":58,"535983":58,"536":58,"536569":58,"537248":58,"537833":58,"538480":58,"539":83,"539074":58,"539724":58,"53k":[52,54,55],"540307":58,"540952":58,"541534":58,"542075":58,"542596":58,"543248":58,"543719":58,"544424":58,"544952":58,"545530":58,"546114":58,"546713":58,"547292":58,"547902":58,"548453":58,"549015":58,"549665":58,"55":55,"550436":58,"551":51,"551925":58,"553105":58,"55c":51,"55k":[52,54,55],"56":[51,52,55,56,83],"560":58,"5620":58,"564":58,"5676":58,"568":58,"57":[55,58],"5746":58,"576":[56,83],"58":[54,55,58],"59":[51,54,55,56,57],"594":51,"597":53,"599":53,"5d":58,"5f":58,"6":[51,52,53,54,55,56,58,60,63,68,80,82,83,85],"60":[52,54,55,57],"600":[56,58],"6047":51,"608":55,"608kb":55,"61":[57,58],"613":58,"62":[51,52,58],"622":[58,60],"62w":55,"62x":53,"63":[51,53,55],"630":[52,54,55],"635":58,"636":58,"637":58,"638":58,"639":58,"64":[53,54,55,58,84],"640":58,"641":58,"642":58,"643":58,"644":58,"6442285180091858":55,"6445754766464233":54,"646":58,"649":58,"64bit":89,"65":[51,52,54,55,58],"6539":58,"655":58,"66":52,"664062":88,"668":51,"669":51,"67":[55,58],"6733":58,"677":58,"67mib":51,"68":[54,58],"6812":[52,54,55],"687":58,"688":58,"689":58,"69":[54,55],"690":58,"6f":[51,53],"6s":55,"7":[51,52,53,54,55,56,58,63,64,80,83,85],"70":[52,54,55,57],"700":[56,58],"701":58,"709":51,"7099":58,"71":[52,55,58],"716":58,"72":[52,54],"7203":58,"72048":85,"721":58,"724":58,"728":51,"729":51,"73":[51,52,54,55],"7302":77,"732":58,"735":58,"7376":58,"738":58,"74":[57,58],"742":58,"7454":58,"75":[52,54,55,58],"7537":58,"76":58,"781":58,"79":[54,58],"796":58,"797":58,"7ubuntu0":51,"8":[3,51,52,53,54,55,56,57,58,60,71,76,77,80,83,85,88,89],"80":[51,52,54,55,57,58],"800":[56,58],"8000":88,"8001":88,"8002":88,"801":58,"81":[57,58],"818":58,"818977576572eadaf62c80434a25afe44dbaa32ebda3a0919e389dcbe74f8656":85,"82":58,"8204":58,"821":58,"83":[52,55,58],"834":58,"8351":58,"837":58,"84":[55,56,58,82,83],"847":58,"84e944ff11f8":[52,54,55],"84x":54,"85":[52,55,58],"86":[52,55],"860":58,"86k":[52,55],"87":58,"8732":57,"877":58,"8791":58,"88":[52,55,57],"89":[52,55],"898":58,"89k":[52,55],"8bit":58,"9":[51,52,53,54,55,56,57,58,80,83,88],"90":[52,54,55,57,88],"900":[56,58],"906":58,"90994":[52,55],"916":[51,58],"91a9cc5850784b2065e8a0aa3d526fd9":51,"92":[52,54,55,88],"9205bed204e2ae7aafd2e01cce0f21309e281e18d5bfd7172ef8541771539d41":85,"9223372036854775807":68,"923":[52,54,55],"927":58,"92k":54,"9367":58,"94":[52,54,55],"941":58,"94328":54,"944":58,"948":58,"94k":[52,54,55],"95":52,"951":53,"952":58,"953":[51,58],"955":51,"959":58,"96":[51,58],"9624":58,"9695423245429993":52,"97":[52,58],"98":58,"9899807572364807":54,"9899841547012329":55,"99":[51,52,53,54,55,57,58],"997":58,"999":58,"9999":58,"99th_p":[51,53],"9ab0":[52,54,55],"9x":51,"abstract":[63,66,77],"boolean":[58,71],"break":[58,76],"byte":[51,71,72],"case":[0,1,2,46,48,53,56,57,59,63,66,85,86,87],"catch":[60,83],"char":[3,4,44,83,89],"class":[17,29,30,44,45,46,50,52,53,54,55,56,57,58,63,66,69,76,77,82,83,84,86],"const":[0,1,2,3,4,29,30,31,32,33,35,37,44,45,46,60,66,68,83,86],"default":[0,1,2,3,4,16,29,30,43,45,46,47,48,51,52,54,55,56,57,61,71,72,74,75,76,83,85,86,89,90],"do":[51,52,54,56,57,59,60,61,66,75,77,82,83,84,86,91],"enum":[0,1,2,42,45,46,50,52,69,72,86],"export":[51,58,85],"final":[51,59,62,64,85],"float":[48,51,52,54,56,57,68,71,82,83,84,86,89,90],"function":[0,1,2,3,4,46,47,48,50,51,52,53,54,56,57,58,60,61,63,66,82,83,85,86,88,90,91],"import":[51,52,53,54,55,56,57,58,60,61,74,76,82,83,84,85,87,88,89,90],"int":[0,3,4,35,44,45,48,52,55,58,68,71,72,74,83,89],"long":[48,53,59,76,77,89],"new":[0,1,2,3,4,32,33,46,47,48,52,54,56,57,58,63,64,66,69,72,76,83,88],"null":51,"public":[0,1,2,3,4,44,45,46,47,48,77,86],"return":[0,1,2,3,4,23,24,29,30,31,32,33,34,37,42,43,44,45,46,51,52,53,54,55,56,58,60,62,63,64,66,69,71,72,82,83,84,86,88],"short":[60,76,77],"static":[47,48,59,66,71,72,74,83],"super":[44,56,82],"throw":[60,83,89],"true":[0,1,2,4,46,48,51,52,53,54,55,56,57,58,60,61,66,68,71,72,74,77,83,86,88,90,91],"try":[51,52,53,54,56,64,76,77,83,90],"var":68,"void":[3,4,25,26,27,28,35,36,42,44,45],"while":[58,85,86,88],A:[4,29,30,32,33,47,51,52,53,54,55,56,58,60,61,66,77,85,86,88],AS:[51,52,53,54,55,56,57,58],And:83,As:[55,83],At:75,But:[76,83],By:[29,30,50,56,57,61,74,82],For:[52,54,55,56,57,58,59,61,74,76,77,82,83,85,86,87,88,90],IS:[51,52,53,54,55,56,57,58],If:[27,51,52,53,54,56,57,58,59,60,69,71,74,76,83,85,86,87,88,91],In:[0,1,2,46,51,52,53,54,55,56,57,58,59,62,63,64,66,67,76,77,79,84,85,86,87,88],Is:[24,71],It:[51,52,53,54,55,56,57,60,61,62,64,66,74,76,85,89],Its:[66,76],NOT:53,No:[52,54,55,56],Not:3,OF:[51,52,53,54,55,56,57,58],OR:[51,52,53,54,55,56,57,58],On:[51,52,54,55,56,61],One:[53,55,76,77,83],Or:76,THE:76,TO:[83,85],That:76,Thats:83,The:[1,46,47,48,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,69,71,72,74,77,82,84,85,86,88,89,90],Then:[61,85,86,90],There:[4,53,57,58,59,64,66,77,82,85,86,87,88],These:[52,53,54,59,63,74,76,86,88],To:[1,46,55,56,57,58,61,74,82,83,84,85,88,90],Will:31,With:[52,53,54,55,74,76,83,86,88],_:[51,52,53,54,55,56,57,58,76],___torch_mangle_10:82,___torch_mangle_4847:63,___torch_mangle_5:82,___torch_mangle_9:82,__and__:68,__attribute__:43,__future__:51,__getitem__:68,__gnuc__:43,__init__:[56,70,71,76,82],__is__:68,__isnot__:68,__not__:68,__or__:68,__range_length:68,__round_to_zero_floordiv:68,__torch__:[63,82,83],__torch___pytorch_detection_ssd_src_model_ssd300_trt_engin:63,__torch___torchvision_models_resnet____torch_mangle_4847_resnet_trt_engin:63,__version__:58,__visibility__:43,__xor__:68,_affin:58,_all_:60,_b:51,_c:[72,90],_calibr:58,_convolut:[58,68,83],_input_quant:58,_jit_intern:51,_jit_to_backend:90,_pair:58,_quant:58,_script:72,_shapemod:71,_theme:81,_trace:51,_validate_not_a_forked_repo:[54,55,57,88],_weight_quant:58,a100:[51,52,53,54,56,57],a1b:77,aarch64:64,ab:68,abi:87,abil:55,abl:[52,53,54,55,56,57,59,60,66,67,86,90],about:[52,54,55,57,58,59,63,66,71,74,83,85,88,89],abov:[25,57,58,69,75,76,83,85],absl:51,absolut:[58,89],absolute_import:51,ac:79,acc:58,acceler:[52,53,54,56,57,91],accept:[47,53,58,63,66,71,83,84,89],access:[55,57,60,66,67,74,83,90],accord:[66,72],accordingli:[58,74],account:88,accumsan:79,accumul:[48,72],accuraci:[57,58,86],achiev:[52,54,56,57,58],aco:68,acosh:68,acoust:51,acquir:83,across:[60,74],acthardtanh:66,action:76,activ:[58,72,76,83,86,91],activationtyp:66,actual:[56,58,60,63,66,69,82,83],ad:[25,59,89],adaptive_avg_pool1d:68,adaptive_avg_pool2d:68,adaptive_avg_pool3d:68,adaptive_max_pool1d:68,adaptive_max_pool2d:68,adaptive_max_pool3d:68,adaptiveavgpool2d:[54,55],add:[26,59,60,61,66,68,69,74,76,81,83,84,85],add_:[60,68,83],add_patch:57,addactiv:66,addit:[55,57,58,60,71,83],addlay:83,address:77,addshuffl:83,adipisc:[77,79],adjac:76,adjust:[58,76],adjust_lr:58,adopt:53,advanc:[77,86],advis:76,aenean:79,affin:[54,55],aforement:88,after:[55,57,58,59,60,61,67,82,83,84,87,88,89],again:[44,53,57,63,66,76],against:[83,89],agre:[51,52,53,54,55,56,57,58],agx:45,ahead:[55,83],aim:[53,60],aiohttp:51,aiosign:51,alabast:51,algo_typ:[70,86],algorithm:[3,4,29,30,44,53,70,86],alias:43,align:76,align_corn:68,aliquam:79,aliquet:[77,79],all:[16,42,43,44,45,48,51,52,53,54,55,56,57,58,60,61,63,69,71,76,77,82,83,84,85,86,87,88,89],alloc:66,allow:[47,48,52,54,56,57,59,60,71,74,89],allow_gpu_fallback:[45,46,71,72,86,90,91],allow_tf32:68,almost:83,alpha:[57,68,77],alreadi:[51,52,53,54,55,56,57,58,59,60,83,86,89],also:[30,48,52,53,54,55,56,57,59,66,67,74,76,77,83,84,85,86],alter:55,altern:47,although:76,altogeth:[61,74],alwai:[3,4,27,76,89],amax:58,amax_sequeez:58,amazonaw:[52,54,55],amet:[77,79],amount:[53,58],amp:[52,54,55],amp_backend:51,an:[2,3,4,47,48,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,67,70,71,72,74,76,77,82,83,84,85,86,87,88,89],analogu:66,analysi:[57,61],analyt:74,analytics_id:74,ancient:76,ani:[47,51,52,53,54,55,56,57,58,59,66,71,74,76,83,84,85,86,89],ann:76,anneal:58,annot:[57,66,83],anonym:76,anoth:[53,76,77,82,84],ant:79,antlr4:51,anyon:77,anyth:[76,77,87],aot:[55,67,83],apach:[51,52,53,54,55,56,57,58],apex:57,api:[51,55,57,58,61,64,66,71,72,75,83,84,86,87,88,90],appdir:51,appear:76,append:[51,52,53,54,55,56,57,58,68],applehelp:51,appli:[58,86],applic:[1,30,46,51,52,53,54,55,56,57,58,60,64,83,84,87,89,90,91],approach:[52,54,56,57],apr:[51,83],apt:51,ar:[42,46,48,51,52,53,54,55,56,57,58,59,60,61,63,64,66,67,71,72,74,76,77,78,82,83,85,86,87,88,89,90],arab:77,arang:68,architectur:[53,57,58,67,85],archiv:[51,54,57,85],arcu:[77,79],area:78,aren:83,arg:[51,55,59,70,71,80,83],argc:83,argmax:[52,53,54,55],argon2:[51,54,55,56,57],argpars:51,argument:[47,51,52,53,58,60,63,66,71,72,76,77,83,89],argv:83,around:[58,60,63,66,76,79,82],arrai:[3,4,33,51,53,59,72],arrayref:[45,47,48],arti:[52,54,55],arxiv:86,as_numpi:88,asin:68,asinh:68,aspect:89,asr:51,asr_model:51,assembl:[59,83],assign:[3,4,75],associ:[53,59,66,83],associatevalueandivalu:66,associatevalueandtensor:[66,83],assum:[51,58,90],ast:51,asttoken:[51,55,57],async:51,asyncio:[51,54,55,56,57],atan:68,atanh:68,aten:[48,57,58,60,61,65,66,68,72,83],atol:89,attach:57,attent:53,attention_mask:53,attention_masks_tensor:53,attr:[51,54,55,56,57],attrdict:[51,88],attribut:[60,61,63,76,83],auctor:79,audio:51,audioread:51,augment:53,augu:79,auth:51,author:77,auto:[44,61,66,76,77,83,86,91],autodoc:[76,77],automat:[52,54,56,57,76,83],av:[54,55,56],avail:[52,54,55,56,57,66,74,85,89,91],averag:[48,52,54,55,56,57,58,72,89],avg:[52,57,58,89],avg_pool1d:68,avg_pool2d:68,avg_pool3d:68,avgpool:[54,55,57,58],avoid:[52,53,54,55],awai:76,await:[52,54,55],awaken:76,ax:[52,54,55,57],axi:[52,54,55,58,68],b0:54,b:[54,55,57,68,77,88],b_hh:68,b_ih:68,babel:51,back:[60,61,63,64,71,76,82,83],back_insert:44,backbon:[53,57],backcal:[51,54,55,56,57],backend:[51,52,53,54,55,56,57,58,72,75,90],background:[76,82],backlink:76,backport:51,backward:58,bar:[74,76],base:[36,49,52,53,54,56,57,58,63,69,70,71,76,82,85,86],basebal:53,baselin:[55,58],bash:85,basi:[51,52,53,54,55,56,57,58,76],basic:[58,77,88,89],batch:[3,4,44,51,52,53,54,55,56,57,58,86,88,91],batch_norm:[66,68],batch_siz:[44,51,53,57,58,86],batched_attention_mask:53,batched_data_:44,batched_indexed_token:53,batched_segment_id:53,batchnorm2d:[54,55],batchnorm:[57,60],batchsiz:51,batchtyp:44,bathroom:76,bazel:[64,85],bazel_vers:85,bazelbuild:85,bazelisk:85,bazelvers:85,bbox:57,bdist_wheel:85,beat:77,beautifulsoup4:[51,55],becaus:[53,66,82,83,85],becom:[53,66],bee:76,been:[59,66,77,83],befor:[48,55,57,58,60,64,66,67,72,83,85,88],beforehand:83,begin:[44,53,76,85],beginn:82,begun:76,behav:[57,78],behavior:[48,57,71,72],behaviour:[51,52,53,54,55,56,57],behind:76,being:[52,54,56,57,83],belong:76,below:[53,57,66,76,83,85,88],benchmark:[52,53,54,56,68],benefit:[66,83],bertformaskedlm:53,bertforpretrain:53,bertforsequenceclassif:53,berttoken:53,besid:76,best:[52,54,55,56,57,76,85],best_result:57,best_results_per_input:57,best_results_per_input_trt:57,beta:68,better:[52,54,56,57,82,86],between:[57,60,66,76,77,85,86],bfe5ad2:52,bia:[53,54,55,56,58,60,68,83],bibendum:79,bibliograph:77,bibtex:51,bidirect:53,bigger:76,bin:85,binari:[44,86],binary_data:88,bind:[3,4,33,44,51,54,55,56,57,72,76],bird:[52,54,55,58,88],bit:[48,53,66,71,72,83],bitbucket:74,bitbucket_url:74,black:[51,57],blandit:79,blank:76,bleach:[51,54,55,56,57],blob:[65,74,86],block0:60,block1:60,block:[59,60,80,89],blue:76,bmm:68,bn1:[54,55],bn2:[54,55],bn3:[54,55],bodi:[76,77],bold:76,bool:[0,1,2,3,4,24,27,29,31,42,44,45,46,48,60,66,68,69,71,72,74,83,86],border:76,bot:57,both:[52,54,56,57,74,76,82,85,86],boto3:51,botocor:51,bottleneck:[54,55],bottom:74,bound:[57,58],box:[57,76],braceexpand:51,bracket:76,branch:[53,85],bread:76,breed:[52,54,55],brief:61,briefli:82,broadli:53,broken:[51,52,53,54,55,56,57],brontosaurus:76,browser:76,bsd:[42,43,44,45],bu:[51,52,54,55,56],buffer:[3,4],bug:85,bui:77,build:[29,30,34,48,51,52,59,62,64,66,71,75,80,83,86,89],build_fil:85,builderconfig:45,built:[33,63,64,72,85,89],bust:[52,54,55],button:[74,76],c10:[0,1,45,46,47,48,83,86],c96b:55,c:[42,43,44,45,51,52,54,55,56,57,58,64,68,77,87,88,89,91],c_api:65,c_str:[66,83],ca6b:[52,54],cach:[3,4,29,30,44,51,54,55,57,58,70,83,86,89],cache_:44,cache_fil:[44,70,86],cache_file_path:[3,4,29,30,44],cache_file_path_:44,cache_size_:44,cachecalibr:[70,86],cachetool:51,cackl:77,cadenc:55,calcuat:58,calcul:[47,59,61,83],calendar:51,calib:58,calib_output:58,calibr:[3,4,29,30,44,48,58,70,72,83,86,89],calibrate_model:58,calibration_cache_fil:[29,30,86],calibration_dataload:[29,86],calibration_dataset:86,calibrationalgo:[70,86],call:[29,30,32,48,51,52,53,54,56,57,58,60,63,66,72,76,82,83,90],callmethod:82,can:[0,1,4,29,30,37,46,47,48,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,71,72,74,76,82,83,84,85,86,87,88,89,90],canada:77,cannot:[47,56,57,58,60,61,71,72,75,82],canon:74,canonical_url:74,cap:[51,52,54,55,56],capabl:[17,45,48,63,71,72,89,90],capit:[53,76],caption:[76,79],captur:58,car:58,card:[52,53,54],cast:[3,4,60],cat:[58,68,85],caught:60,caus:[52,54,56,57,58,66,74,85],cd:[85,88],cdll:83,ceil:68,ceil_mod:[54,55,68],cell:[53,57,77],center:[52,53,54],centercrop:[52,54,55,88],cerr:83,certain:[51,85],certifi:[51,53],cf0691493d05062fe3239cf76773bae4c5124f4b039050dbdd291c652af3ab2a:85,cffi:[51,54,55,56,57],cfg:61,chain:66,challeng:[58,88],chanc:66,chang:[30,52,53,54,55,56,57,60,64,74,86,88],changelog:80,channel:[2,52,54,55,58,71,75],channel_last:[55,71,72],channels_last:71,charact:76,charset:[51,53],check:[0,1,31,46,55,57,60,66,72,83,85,87,88,89],check_method_op_support:72,check_method_operator_support:[21,41,45,49],checkmethodoperatorsupport:83,checkpoint:[51,53,54,57,58],child:77,chimpansee_amber_r_1920x1080:[52,54,55],chimpanze:[52,54,55],choic:[53,70],choos:[52,54,82],ci:[51,52,54,55,56],cifar10:[58,86],cifar:[58,86],circular:58,ckpt:58,ckpt_path:58,cl:53,clamp:[58,68],clamp_max:68,clamp_min:68,class_count:88,class_pr:58,class_prob:58,classes_to_label:57,classif:[56,57,58,82,83],classifi:[56,58,77],classification_index:88,clean:76,clear:44,cli:89,clib:51,click:[51,53,57],clickabl:76,client:[51,54,55,56,57],clone:68,close:[58,83],closer:60,closet:76,cloud:51,cloudfront:[52,54,55],co:[68,77],coco:57,cocodataset:57,code:[52,53,54,55,56,57,61,64,67,75,77,82,83,86],collapse_navig:74,collat:77,collect:[51,52,54,56,57,58,83],collect_stat:58,colon:76,color:[24,27,69,76],colorama:51,colored_output_on:[27,42,69],column:77,com:[51,52,53,54,55,56,57,65,83,85,86,87,88],come:[52,54,55,56,57,75,85,88],command:[76,77,82,83,85,88,89],comment:[76,85],commodo:79,common:[51,54,57,59,60,76],common_subexpression_elimin:60,commonli:77,commun:83,compani:53,compar:[53,57,58],comparis:[0,2],comparison:[1,46],compat:[0,1,46,52,54,56,57,60,63,72,85],compil:[21,31,37,41,45,48,49,51,52,53,54,55,56,57,58,60,61,63,66,69,71,72,74,82,84,86,87,88,89,90,91],compile_set:[51,86],compile_spec:[58,86,91],compilegraph:[83,86],compilesepc:33,compilespec:[3,4,21,32,37,41,45,49,61,83,86,91],compilespecstruct:49,complet:[51,52,53,54,56,57,61,82,83],complex:[82,85],compli:57,complianc:[51,52,53,54,55,56,57,58,89],compliat:86,complic:85,compon:[53,56,62,64,82,87],compos:[52,54,55,56,57,58,82,86,88],composit:[58,83],comprehens:57,compris:53,comput:[48,51,52,53,54,55,56,57,58,76,85,86],compute_amax:58,conceiv:76,concern:53,conclus:[51,52,53,54],concorr:88,conda:[51,52,53,54,55,56,57,58],condimentum:79,condit:[51,52,53,54,55,56,57,58,76],conduc:55,conduct:53,conf:[74,81],confid:[52,54,55,57],confidence_scor:88,config:[51,52,85,88],configur:[32,37,47,51,55,67,71,72,80,83,85,86,88],confirm:51,conflict:[51,52,53,54,55,56,57],congu:79,connect:[52,54,55,60,72,76,88,91],consectetur:[77,79],consecut:61,consid:[55,83],consider:88,consist:[53,60,76],consol:89,consolid:82,constant:[55,58,59,60,83],constant_pad_nd:68,constexpr:[0,1,2,45,46],construct:[0,1,2,3,4,46,47,48,59,60,62,64,66,70,71,76,77,83,86],constructor:[0,2,46,47,48,63,82],consult:75,consum:[4,59,82],contact:77,contain:[29,31,51,52,53,54,55,56,57,59,60,66,71,76,77,82,83,85,86,87,88],content:[55,80,86,88],context:[52,56,58,59,62,63,64,69],contextnet:51,contigu:[2,47,48,71,72,89],continu:[52,53,54,56,57,76,87],contributor:83,control:[56,57,82],conv1:[54,55,56,82,83],conv2:[54,55,56,82,83],conv2d:[54,55,56,58,82],conv3:[54,55],conv4_x:57,conv5_x:57,conv:[48,58,83,89],conv_asr:51,conval:79,convect:47,conveni:[57,86],convent:[52,53,54,56,57],convers:[52,56,57,58,60,61,63,71,72,83],conversionctx:[66,83],convert:[3,4,31,32,37,51,52,54,55,56,57,58,60,61,62,64,67,71,72,84,87,90],convert_method_to_trt_engin:[21,41,45,49,71,72,90],convertgraphtotrtengin:83,convien:48,convienc:[3,4,48],convnet:57,convolut:[51,52,55,57,58,72,86,91],coordin:64,copi:[44,51,52,53,54,55,56,57,58,66,68,70,77,88],copy_:68,copyright:[42,43,44,45,51,52,53,54,55,56,57,58,77,83],core:[45,51,52,54,55,56,57,60,61,64,71,83,89,91],core_id:71,corpor:[42,43,44,45,51,52,53,54,55,56,57,58],correct:[58,63,74,85],correctli:85,correspond:[57,58,66],cosh:68,count_include_pad:68,counterpart:58,coupl:[52,54,56,57,59,64,87],cout:83,cp38:57,cp:85,cpp:[14,15,42,43,44,45,50,60,64,83,86],cpp_frontend:86,cppdirectori:49,cppdoc:83,cpu:51,cra:79,creat:[29,30,33,51,52,53,54,55,56,57,58,59,63,66,72,76,83,88,89],create_model:52,create_transform:52,creating_torchscript_module_in_python:84,credit:83,crit:58,criteria:[61,62,64],cross:[58,76],crossentropyloss:58,cs:86,csrc:[60,65],cstddef:86,ctc_bpe_model:51,ctx:[66,83],ctype:83,cu102:85,cuda113:85,cuda:[48,51,52,53,54,55,56,57,58,63,71,83,84,85,86,88,90],cuda_runtim:[21,45],cudafloattyp:83,cudasetdevic:35,cudnn8:85,cudnn:[51,52,53,54,55,56,57,58],cudnn_en:68,cumsum:68,curabitur:79,curl:[76,85],current:[23,52,54,63,66,72,74],cursu:79,custom:[52,54,85],cut:76,cxx11:87,cycler:51,cython:51,d17fnq9dkz9hgj:[52,54,55],d:[51,52,53,54,55,56,57,58,76,77,89,91],dapibu:79,data:[0,2,3,4,29,30,44,46,47,48,51,52,53,54,56,57,58,59,61,62,64,66,68,70,71,72,76,80,86,89],data_dir:86,data_item_1:75,data_load:58,data_typ:88,databas:51,dataflow:[66,83],dataload:[4,29,30,44,48,58,70,86],dataloader_:44,dataloadercalibr:[70,86],dataloaderopt:86,dataloaderuniqueptr:[4,44],dataset:[30,57,58,70,86],datatyp:[1,21,38,45,46,47,48,49,52,71,72,84,88],datatypeclass:49,date:77,dateutil:[51,54,55,56,57],david:77,dbg:85,ddof:[51,53],dead_code_elimin:60,deal:66,debian_frontend:51,debug:[16,27,45,48,58,66,69,72,89,90],debugg:[72,89],debugpi:[51,54,55,56,57],decid:[56,71],declar:[58,85],decod:[52,53,54,55],decode_result:57,deconvolut:91,decor:[51,54,55,56,57],dedic:[60,77],deep:[52,53,54,55,56,57,58,66,67,74,86,91],deeplearn:65,deeplearningexampl:57,deer:58,def:[51,52,53,54,55,56,57,58,76,82,88],default_tim:[51,53],defer:55,defin:[0,1,2,3,4,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,43,46,47,48,50,51,53,55,56,58,71,74,82,83,84,86,89],definit:[50,66,76],defusedxml:[51,54,55,56,57],deiti:76,delet:[0,1,2,45,46,60],delimit:60,demo:[52,53,54,57,76,86],demonstr:[51,52,53,54,55,56,57,76,77,78,86,88],demonstrat:[52,54],denorm:57,denot:[53,76],dep:85,depend:[30,34,51,55,57,58,59,61,64,83,87,88],depickl:63,deploi:[52,54,55,56,57,62,64,67,83,86,88],deploy:[52,54,56,57,58,83,84,86,87,88,89,91],deprec:[51,68],depth:74,dequantizelay:58,descclassnam:76,descnam:76,describ:[48,54,56,57,66,82,88,90],descript:[55,61,77],deseri:[71,72,83],design:[52,53,54,56,57,91],desir:[58,77,86],destini:77,destroi:[66,77],destructor:66,detail:[52,54,55,58,82,83,87,88],detect:[47,54,58,63],detections_batch:57,determin:[52,60],determinist:68,develop:[51,52,53,54,56,57,67,76,77,83,85],devhelp:51,deviat:89,devic:[21,33,35,38,45,48,49,52,54,56,57,58,63,68,70,71,72,84,86,89,90,91],device_typ:[45,46,71,86,90,91],deviceclass:49,devicetyp:[21,38,45,46,49,71,86,90,91],devicetypestruct:49,diam:79,dict:[57,71,72],dictionari:[53,71,72,90],dictum:79,dictumst:79,did:76,didn:76,differ:[30,53,55,56,57,58,60,64,67,74,82],differenti:[52,54,56,57],digit:51,dignissim:79,dilat:[54,55,56,57,58,68],dim0:68,dim1:68,dim:[52,54,55,58,68,88],dim_int:68,dim_intlist:68,dimens:[47,55,60],dir:58,direct:[80,87],directli:[66,67,70,85,86],directori:[18,19,20,21,42,43,44,45,49,58,85,86],disabl:[52,54,55,56,57,58,69,74,75,85,89],disable_calib:58,disable_qu:58,disable_tf32:[45,48,72,86],disclos:85,disconnect:76,discret:76,discuss:[55,88],disp:[51,52,54,55,56],displai:[69,74,89],display_github:74,display_gitlab:74,display_vers:74,disregard:52,dist:85,distanc:51,distdir:85,distribut:[51,52,53,54,55,56,57,58,71,83,86,87],div:68,div_:68,divis:51,divisor_overrid:68,django:75,dl:76,dl_open:87,dla:[1,45,46,67,71,72,89],dla_cor:[45,46,71,86,89,90,91],dla_standalon:89,dlacor:89,doc:[58,64,65,74,75,76,81,85],docker:[51,52,53,54,56,57,88],docopt:51,docsrc:64,docstr:[76,77],document:[42,43,44,45,49,52,54,55,64,74,76,77,81,82,83,86,87,88,90],docutil:[51,76,77],doe:[43,44,53,57,60,61,66,76,86],doesn:[58,76,82,83],dog:58,dolor:[77,79],domain:[77,86],don:[56,66,74,76,77,86,88],done:[51,55,57,59,61,64,88],donec:[77,79],dont:42,dothismethod:76,dotpai:75,dotpayprovid:75,doubl:[48,72,76,89],down:[52,54,56,57,74,85],download:[51,52,54,56,57,58,80,85,86,88],downsampl:[54,55],doxygen_should_skip_thi:[44,45],dpython:[71,72],dream:77,driver:[51,52,54,55,56,85],drop:[57,74,85],dt:76,dtype:[45,47,48,51,52,53,54,55,56,57,58,68,71,72,84,89],dual:76,due:[3,4,52,54,56,57,58,75,76,85],dui:[77,79],dummi:53,dump:[36,85,89],dump_build_info:[21,38,45,49,71],durat:76,dure:[48,58,66,70,86,87,89],dynam:[47,48,57,58,71,72],e1109:58,e:[29,30,52,53,54,57,60,66,71,82,83,85,86,89],each:[3,4,48,53,57,58,59,60,61,63,66,74,76,83,85],eager:[52,54,56,57],ear:76,earliest:58,eas:43,easi:[59,60,83,86,89],easier:[53,58,62,64,66,83,86],easiest:85,easili:[3,4],ecc:[51,52,54,55,56],echo:76,ecosystem:[52,54,56,57],edg:76,edgecolor:57,edit:74,editdist:51,edu:86,effect:[51,58,60,74,83,86],effici:66,efficientnet:[54,57],efficientnet_b0:52,efficientnet_b0_model:52,efficientnet_preprocess:52,efficitur:79,effort:55,eg:88,egesta:79,eget:79,either:[47,48,51,52,53,54,55,56,57,58,66,71,72,74,76,82,83,85,89],el:68,eleifend:77,element:[53,63,76,77,80],element_typ:44,elementum:79,elit:[77,79],elk:76,els:[43,44,47,51,58,72,76,77],elu:68,emb:[33,72,77,89],embed:[63,68,72,76,89,91],embed_engine_in_new_modul:[21,41,45,49,72],emit:59,emphasi:76,emploi:53,empti:[48,56,72,77,82],emum:[16,17],en:[51,74],enabl:[3,4,24,48,52,54,55,56,57,58,61,62,64,69,70,72,74,89],enable_calib:58,enable_precis:83,enable_qu:58,enabled_precis:[45,48,51,52,53,54,55,56,57,58,71,72,83,84,86,88,90,91],enalbed_precis:91,enc:53,enc_input:53,encdecctcmodelbp:51,encod:[51,53,63],encoded_input:53,encorag:[52,53,54],encount:85,encourag:[55,88],end:[44,66,68,72,76,83,86,89],end_dim:[68,83],end_tim:[51,52,53,54,55,56,57,58],endif:[43,44,45],energi:76,enforc:83,engin:[0,1,17,32,33,37,45,46,47,48,51,53,55,59,61,62,64,67,71,72,74,83,84,86,87,89,90,91],engine_converted_from_jit:83,enginecap:[21,38,45,48,49,71,72,90],english:53,enhanc:[57,76],enim:79,enjoi:53,enough:58,ensur:[30,58,60,61],enter:[53,59],entir:[58,76],entiti:76,entri:[48,66],entropi:[29,30,58,86],entropy_calibr:70,entropy_calibration_2:[70,86],entrypoint:[51,54,55,56,57],enumer:[0,1,2,16,17,46,53,58,70],environ:[51,52,53,54,55,56,57,88],ep:[54,55,68],epoch:58,eq:[68,76],equat:76,equival:[32,56,57,62,64,66,72,82,83,86],equivil:37,erat:79,erf:68,eric:76,ero:79,error:[16,48,51,52,53,54,56,57,59,60,64,69,72,76,83,85,89],eskimo_dog:52,essenc:76,essenti:55,est:79,et:79,eta:[52,54,56,57],etc:[74,76,91],etiam:79,eu:79,euismod:79,eval:[51,52,54,55,56,57,58,83,84,88],evalu:[57,62,63,64],evaluated_value_map:[59,66],even:83,event:47,everi:[61,83],everyth:16,ex:[0,1,2,33,46,72,77,79],exact:88,exactli:[53,57],examin:53,exampl:[47,52,54,55,56,57,58,61,63,64,66,69,71,72,74,75,77,80,82,83,86,87,88],exceedingli:76,except:[51,52,53,54,55,56,57,58],exception_elimin:60,excerpt:77,excit:51,execpt:60,execut:[33,51,52,54,55,56,57,60,62,63,64,71,72,82,83,86,88,89],execute_engin:[63,83],exert:76,exeuct:63,exhaust:83,exist:[4,31,32,37,51,70,71,72,85,86],exit:88,exp:68,expand:[60,68],expand_a:68,expanded_pad:58,expect:[47,48,52,53,54,55,56,57,60,66,71,83],experi:[52,53,54,56,57],experiment:58,explic:[44,58],explicit:[0,1,2,3,4,45,46,55,60,67,76,86],explicitli:[53,58,61,62,64,86,90],explict:44,explictli:0,expon:68,export_util:51,expos:86,express:[51,52,53,54,55,56,57,58,76],ext:[76,77],extend:[51,62,64,66,68,83],extens:[51,53,55,57],extent:[67,83],extern:[74,76],extra:[48,83],extract:83,extractor:56,extrem:76,ey:76,f16:[83,89,91],f1:[52,54,55],f32:89,f:[51,56,58,76,82,85],facecolor:57,facilisi:79,fact:85,facto:76,factori:[4,29,30,86],fail:[83,91],fake:58,fake_quantize_per_:58,fake_quantize_per_channel_affin:[58,68],fake_quantize_per_tensor_affin:[58,68],fall:71,fallback:[62,64,66,89,91],fals:[0,1,2,3,4,44,45,46,48,51,54,55,57,58,68,71,72,74,75,76,77,83,86,90],fame:79,famili:[52,54,56,57,58],familiar:88,familyhandyman:[52,54,55],fan:[51,52,54,55,56],far:76,fashion:83,faster:58,fastjsonschema:55,fasttext:51,faucibu:79,fbed:[52,54,55],fc1:[56,82,83],fc2:[56,82,83],fc3:[56,82,83],fc:[48,54,55,57,58,60,89],feat:[56,82,83],featur:[51,52,53,54,55,56,57,58,61,83,86,89,90],feb:[52,54,56],fed:[3,4,47],feed:[29,30,58,83],feel:[55,67],feli:79,feugiat:[77,79],few:[52,54,56,57,71],ffedb78:76,ffmpeg:51,field:[3,4,86],fifth:77,fig:[52,54,55,57],figur:[61,77,79],file:[0,1,2,3,4,5,6,7,8,9,10,11,12,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,51,52,53,54,55,56,57,58,61,63,64,70,71,72,74,75,77,81,83,85,86,88,89],file_path:89,filelock:[51,53],filer_publ:[52,54,55],filer_public_thumbnail:[52,54,55],fill:[51,52,53,54,56],filter:[51,57,58],find:[4,53,57,83],fine:[51,58],finetun:58,finibu:79,finish:57,first:[47,51,53,56,57,58,59,60,76,77,83,84,86,88],firstli:88,fit:76,five:57,fix:[48,54,57,76,91],fixed_s:[45,48],flag:[58,61,62,64,70,85,87,89],flatten:[56,68,82,83],flatten_convert:83,flesh:88,flexibl:[52,54,56,57],float16:[51,52,54,56,57,71,89],float32:[47,48,51,52,53,54,55,58,71,72,89],float64:72,float_int:68,floor:68,floor_divid:68,floordiv:68,flow:[56,57,58,66,76,82],flox:76,fluent:53,flush:76,fly:82,fmax:51,fmin:51,focal:51,fold:77,follow:[33,51,52,53,54,55,56,57,58,61,63,72,74,76,77,81,82,83,85,86,87,88,89],fonttool:51,foo:[76,77],footprint:[52,54,56,57],forc:[72,74,89],forced_fallback_op:61,form:[51,53,59,71,76,88],format:[33,45,47,48,51,52,53,54,55,56,57,58,68,71,72,76,77,84,88,89],forth:77,forum:85,forward:[29,30,32,33,56,58,61,63,66,71,72,82,83,86,90],found:[42,43,44,45,51,52,54,55,56,57,76,83,85,86,87],four:[76,77],fp16:[0,47,48,53,55,56,57,58,67,83,84,89,91],fp32:[0,47,48,53,55,56,57,58,67,72,86,88,89],frac:76,framework:[52,54,56,57],franc:53,freed:66,freeli:55,freeze_modul:60,fri:52,friend:45,fringilla:79,frog:58,from:[0,1,2,3,4,29,30,44,46,47,48,51,52,53,54,56,57,58,59,60,61,62,63,64,66,67,72,74,75,76,77,82,83,86,88,89],from_pretrain:[51,53],frozen:58,frozendict:51,frozenlist:51,fssl:85,fsspec:51,fstream:[20,44],full:[48,58,66,69,83,86,87,88,89,91],fulli:[31,60,72,83,86,89,91],fusc:79,fuse:[52,54,56,57],fuse_addmm_branch:60,fuse_flatten_linear:60,fuse_linear:60,fusion:66,futur:[51,52,53,54,56,58],futurewarn:51,g2p:51,g:[29,30,51,53,60,71,76,85,86,89],g_:76,gain:57,game:53,gamma:68,gatewai:75,gaurd:43,gcc:[64,83],gdown:51,ge:68,gear:86,geforc:[52,54,56],gener:[3,4,30,51,52,53,54,55,56,57,58,60,63,64,66,74,76,77,80,82,83,85,86,89],genutil:[51,54,55,56,57],geometr:53,get:[0,1,2,3,4,23,34,44,46,57,58,60,61,66,69,71,85,86,88],get_batch:70,get_batch_impl:44,get_batch_s:70,get_build_info:[21,38,45,49,71],get_cache_mode_batch:70,get_coco_object_dictionari:57,get_is_colored_output_on:[18,39,42,49,69],get_logging_prefix:[18,39,42,49,69],get_model_size_mb:51,get_reportable_log_level:[18,39,42,49,69],getattr:[51,60,63,82,83],getbatch:[3,4,44],getbatchs:[3,4,44],getdimens:[66,83],getoutput:[66,83],gi:[51,52,54,55,56],git:80,gitdb:51,github:[51,52,53,54,56,57,65,74,83,85,86,87,88],github_url:74,gitlab:74,gitlab_url:74,gitpython:51,give:[56,74,76],given:[47,48,53,57,60,70,71,72,82,83,89,90],global:[26,58,83],gnu:85,go:[44,52,54,55,56,57,58,60,61,67,82,83,88],goal:66,goe:[58,76],good:[44,66,76],goodger:77,googl:[51,53,74],got:[76,83],govern:[51,52,53,54,55,56,57,58],gpu:[1,32,35,37,45,46,51,52,53,54,55,56,57,58,71,72,83,86,88,89,90,91],gpu_id:[35,45,46,71,86,89,90,91],granular:56,graph:[16,31,32,37,45,51,52,54,55,56,57,58,59,61,62,64,66,67,69,72,82,83,89],graphic:55,graphnam:[51,53],gravida:79,great:[52,54,56,57,76,83],greater:69,green_mamba:[54,55],group:[58,68,76,77],grpc:88,grpcio:51,gru_cel:68,gt:[51,52,53,54,55,56,57,68],gtc:67,guangzhou:77,guard:60,guard_elimin:60,guess:53,gui:76,guid:75,gulf:[52,54,55,88],gz:[76,77,85,86],h5py:51,h:[0,1,2,3,4,5,6,7,8,9,10,11,12,15,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,49,50,52,54,57,60,83,86,89],ha:[51,52,53,54,55,56,57,58,59,60,61,62,64,66,76,77,82,83,86],habit:79,habitass:79,hac:79,hack:44,hakaimagazin:[52,54,55,88],half:[53,55,56,57,58,71,76,83,84,86,88,89,90,91],hand:88,handl:[57,60,63],happen:[56,58,82],hardtanh:[66,68],hardtanh_:68,hardwar:[52,54,56,57,91],hasattr:51,hash:85,have:[30,33,44,51,52,53,54,55,56,57,59,60,66,67,71,72,76,82,83,85,86,88,89],haven:83,head:57,header:[52,54,55,74,76,77,83,88],heart:77,heaven:76,heck:76,heh:77,hehe:77,height:76,help:[27,51,52,53,54,56,58,59,66,83,87,89],helper:[51,56,57,58,66],henc:53,hendrerit:79,here:[44,51,52,53,54,55,56,57,59,61,63,74,76,77,82,83,85,86,87,88],hermet:85,hexagram:76,hfile:49,hi:[68,76,77],hidden:[43,53,74],hierarchi:58,high:[52,54,57,60,61,74],higher:[53,60,74,76,82],highfreq:51,highli:[55,88],highlight:76,hinton:86,hist_percentil:58,histogram:58,historgram:58,hit:51,hold:[46,47,59,66,86],holder:[63,78],holi:76,home:85,hood:[64,84],hope:77,hors:58,host:[54,55,56,57,58,85,88],how:[3,4,52,53,54,55,57,58,76,78,80,82,87,88,90],howev:[30,52,54,56,57,74,75,85,88],html:[58,65,76,82,85,86],html_theme:81,html_theme_opt:74,html_theme_path:81,htmlhelp:51,http:[51,52,53,54,55,56,57,58,65,74,76,82,83,85,86,87,88],http_archiv:85,httpclient:88,hub:[51,53,54,57,88],huge:53,huggingfac:[51,53],human:76,humankind:77,huski:[52,54,55],hx:68,hybrid:72,hydra:51,hyperlink:76,hyphen:76,i8:89,i:[51,52,53,54,55,56,57,58,60,66,76,77,82,83,86,89],iaculi:79,icon:[74,76],id:[35,45,51,52,54,55,56,57,71,74,75,79,89,91],idea:[60,76],ident:[53,89],idna:[51,53],idx:[57,68],ifndef:[44,45],ifstream:44,ignor:71,iii:77,iint8calibr:[3,4,29,30,44,45,48,72,86],iint8entropycalibrator2:[3,4,29,30,44,86],iint8minmaxcalibr:[29,30,86],ilay:66,illustr:58,imag:[52,54,55,57,58,86,88],image_classif:57,image_idx:57,imageio:57,imagenet:[52,54,55,58],imagenet_cla:[52,54,55],imagenet_class_index:[52,54,55],images:51,images_:86,img0:[52,54,55],img1:[52,54,55,88],img2:[52,54,55],img3:[52,54,55],img:[52,54,55,88],img_path:[52,54,55,88],impact:[52,53,54,56,57],imperdiet:79,implement:[3,4,52,53,54,55,56,57,60,61,63,75,83,86,87],impli:[51,52,53,54,55,56,57,58],implic:60,implicit:[68,76],implicitli:71,implictli:71,importlib:[51,54,55,56,57],improv:[58,77],imshow:[52,54,55,57],in_featur:[54,55,56],in_shap:83,in_tensor:82,incas:44,includ:[13,15,16,34,36,42,43,44,45,50,52,54,56,57,61,62,63,64,74,76,82,83,85,86,89],includedirectori:49,includehidden:74,incompat:85,incorpor:77,incorrect:58,ind:[52,54,55],inde:[52,54,56,57],indent:76,independ:57,index:[33,51,52,53,54,55,56,57,58,65,67,68,72,74,80,86],indic:[51,53,68,74,76],indigo_bunt:52,indirect:76,inetworkdefinit:59,infer:[51,52,53,54,56,58,60,71,72,83,86],inference_output:88,inferenceservercli:88,inferinput:88,inferrequestedoutput:88,inflect:51,info:[16,32,37,45,48,66,69,71,83,89],inform:[25,33,34,36,47,51,55,57,58,59,61,63,67,69,71,76,82,83,85,86,89,90],infrastructur:[86,88],ingest:64,inherit:[49,86],iniconfig:51,init_weight:58,initi:[51,53,58,76],injuri:76,inlin:[0,1,2,3,4,29,30,44,46,48,51,54,55,56,57,60,77,80,83],inner:[48,77],innings:53,inplac:[54,55],input0:83,input1:83,input2:83,input:[3,4,21,30,33,38,44,45,48,49,51,52,53,54,55,56,57,58,59,60,61,63,66,68,69,71,72,77,82,83,84,86,88,89,90,91],input_0:[63,83],input__0:88,input_batch:[52,54,55],input_data:[52,54,55,56,57,58,82,84],input_file_path:[89,91],input_id:53,input_is_dynam:45,input_s:[61,83],input_scal:68,input_shap:[51,52,54,55,56,57,58,86,91],input_spec:89,input_tensor1:53,input_tensor2:53,input_tensor3:53,input_tensor:[51,52,54,55],inputclass:49,inputrang:[61,83],inreleas:51,insert:[58,83,86],insid:[76,88],inspect:[52,54,56,57,66,82,83],instal:[51,52,53,54,55,56,57,58,67,80,83,87,88],instanc:[53,56,60,70,82,83],instance_norm:68,instanti:[51,62,63,64,66,83],instatin:[0,1,2,46],instead:[48,51,52,53,54,55,56,57,58,59,60,83,87,89],instnanti:63,instruct:[61,62,64,83,85,88],insur:85,int32:[53,55,71,72],int64:72,int64_t:[45,46,47,48,86,91],int8:[0,44,47,48,55,67,71,72,86,89,91],int8_t:[17,45],int8cachecalibr:[20,30,40,44,49],int8cachecalibratortempl:49,int8calibr:[3,20,29,40,44,49],int8calibratornamespac:49,int_float:68,integ:[58,71,79],integr:[52,53,54,55,56,57,67],intend:[51,85],intent:[60,76],interact:76,intercompat:57,interdum:79,interest:[60,76],interfac:[0,1,2,46,63,64,66,86],interfer:76,intermedi:[16,52,54,56,57,69,82],intern:[1,16,46,52,54,56,57,58,66,69,76,83],internal_error:69,internalerror:69,interpol:[52,76],interpolationmod:52,interpret:[52,54,56,57,63,76],intro_to_torchscript_tutori:82,introduc:[52,54,56,57,58],introduct:53,invalid:58,invok:[82,83],involv:[51,52,53,54,56],io:[44,51,52,53,54,55,56,57,88],iostream:[20,21,44,45,83],ipad:51,ipso:76,ipsum:[77,79],ipykernel:[51,54,55,56,57],ipython:[51,54,55,56,57],ipywidget:[51,54,55,56,57,58],ir:[52,54,56,57,62,64,66,71,82],is_avail:[52,54,55],is_floating_point:68,is_tar:51,is_train:86,iscustomclass:66,isinst:58,isn:[74,76],isort:51,issu:[3,4,51,52,53,54,56,83,85],istensor:66,istream_iter:44,it_:44,ital:76,item:[51,52,53,54,55,58,75,77],itensor:[59,66,83],iter:[20,44,48,51,52,53,54,55,56,57,58,59,70,72,89],its:[30,52,54,56,57,59,63,66,76],itself:[0,1,2,46,60,85,88,89,90],iv:77,ivalu:[59,63,66,83],ja:51,jan:77,jarowinkl:51,jedi:[51,54,55,56,57],jetpack:85,jetpack_4:85,jetson:[52,54,56,57,71],jieba:51,jinja2:[51,54,55,56,57],jit:[31,32,33,37,45,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,71,72,82,83,84,88,89,90],jit_model:58,jmespath:51,joblib:[51,53],join:58,jpeg:[52,54,55],jpg:[52,54,55,57,88],jpg__1920x1080_q85_subject_loc:[52,54,55],jsmath:51,json:[52,54,55],json_fil:[52,54,55],jsonschema:[51,54,55,56,57],jump:88,jupyt:[51,54,55,56,57],jupyterlab:[51,54,55,56,57],jupyterlab_widget:[54,56,57],just:[44,45,52,53,54,55,57,60,67,69,76,78,82,83,84,87,90],justo:[77,79],k:[53,68,86],kaldi:51,kaldiio:51,kb:[52,54,55,56,57],kbool:[0,45],kchannelslast:[2,45],kchar:[0,45],kclip:66,kcontigu:[2,45,47],kcpu:[1,46],kcuda:[1,46,61,83],kdebug:[16,42,44],kdla:[1,45,46,91],kdla_standalon:[17,45],keepdim:68,kei:[53,58,76,82,88],kept:[58,77],kernel:[47,48,52,54,56,57,66,71,72,89],kernel_s:[54,55,56,68],kerror:[16,42],keyboard:76,keyword:[51,71,72],kf16:[86,91],kfloat:[0,45,48],kgpu:[1,45,46],kgraph:[16,42,60],khalf:[0,45,83],ki8:86,kind:[51,52,53,54,55,56,57,58,59,71],kinfo:[16,42,44],kint:[0,45],kinternal_error:[16,42],kiwisolv:51,know:[42,66,74,76],knowledg:76,kriz:86,krizhevski:86,ksafeti:[17,45],kstandard:[17,45,48],ktest:86,ktrain:86,kunknown:[0,2,45],kwarg:[55,58,70,71],kwarn:[16,42],l:68,label:[52,54,55,57,58,76,86,88],lacinia:79,lack:[61,62,64],lacu:79,laid:83,lambda:[54,55,57,66,76,83,88],lang:75,languag:[51,52,54,55,56,57,58,75,76,77,82,88],laoreet:79,larg:[52,53,54,56,57,58,62,64,74,76,83,86],larger:[74,86],largest:68,last:[2,51,57,60,71],lastli:88,latenc:[51,53],later:[30,53,83],latest:[52,53,54,74,85],latexcodec:51,launch:88,law:[51,52,53,54,55,56,57,58],layer1:[54,55],layer2:[54,55],layer3:[54,55],layer4:[54,55],layer:[46,48,52,54,56,57,58,59,60,66,72,83,86,88,89,91],layer_norm:68,layout:[2,47,68,71,72],ld_library_path:85,ld_preload:87,ldd:85,le:68,lead:76,leader:76,leaky_relu:68,leaky_relu_:68,learn:[55,58,67,83,85,86,88,91],leas:77,least:[52,53,54,76,77],leav:[56,58,60],lectu:[77,79],left:[57,74,76],legacy_calibr:70,legend:76,len:[51,53,57,58,68],lenet:[82,83],lenet_script:[82,83],lenetclassifi:[56,82],lenetfeatextractor:[56,82],length:[3,4,44,52,53,54,55,68,77],leo:79,let:[46,51,52,54,55,56,57,60,66,71,72,74,76,88,89],letter:[51,77],level:[18,23,25,26,39,42,44,49,52,53,54,56,57,58,60,61,64,69,80,82,88],levelnamespac:49,leverag:[52,54,55,56,57,86],lib:[51,52,53,54,55,56,57,58,60,83,85],libero:[77,79],librari:[34,42,43,44,45,52,54,55,56,57,58,62,63,64,66,83],librosa:51,libsndfile1:51,libtorch:[4,36,52,54,56,57,66,83,85,86],libtorch_pre_cxx11_abi:85,libtorchtrt:[83,85,89],libtorchtrt_plugin:87,libtorchtrt_runtim:87,licens:[42,43,44,45,51,52,53,54,55,56,57,58,83],light:76,lightn:51,lightningdeprecationwarn:51,lightningmodul:51,ligula:79,like:[52,54,56,57,59,60,63,66,75,76,82,83,84,85,86,87,88,89],limit:[51,52,53,54,55,56,57,58,60,69,75,86],line:[77,83,89],linear:[2,54,55,56,58,68,71,82],linewidth:57,link:[59,67,74,75,80,83,87,89],linux:[64,83,85],list:[18,19,20,21,31,48,50,51,53,57,58,59,61,63,66,68,71,72,80,83,84,85,88],listconstruct:[59,63,83],listunpack:[63,83],liter:77,literal:77,literal_block:76,live:[66,76],ll:53,llvmlite:51,lo:68,load:[51,52,54,55,57,58,61,63,70,72,83,84,86,87,88,89,90],load_calib_amax:58,load_librari:87,load_state_dict:58,loader:[51,52,54,56,57],loading_data_recip:86,loborti:[77,79],local:[57,58,60,74,83],localhost:88,locat:[57,85,86],lock:75,log:[15,16,19,20,38,44,49,50,53,58,60,66,67,68,71],log_debug:66,logger:69,loggingenum:49,login:88,loglevel:69,logo_onli:74,lone:77,longer:[52,54,56,57,74,87],look:[51,52,53,54,55,56,57,58,59,60,82,86,88,90],loop_unrol:60,lorem:[77,79],lorikeet:[54,55],lose:74,loss:[58,86],lot:66,lower:[16,55,69,71,77],lower_graph:60,lower_tupl:60,loweralltupl:60,lowersimpletupl:60,lowfreq:51,lr:58,lstm_cell:68,lt:[51,53,54,55,56,57,58,68],ltorchtrt:87,luctu:79,lvl:[25,26,42],m:[51,52,54,55,56,77],machin:[52,54,56,57,63,85,86,88],macro:[5,6,7,8,9,10,11,12,15,18,21,42,45,49,50],mad:76,made:[57,60,62,64,76],maecena:79,magna:79,mai:[51,52,53,54,55,56,57,58,59,63,64,76,77,82,83,85,86,88],main:[57,60,61,62,63,64,66,74,76,78,83],maintain:[53,61,63,66],major:[52,54,56,57,64],make:[52,53,54,55,56,57,59,76,78,83,84,85,86,88,91],make_data_load:[4,86],make_int8_cache_calibr:[20,40,44,49,86],make_int8_calibr:[20,30,40,44,49,86],malesuada:79,man:[76,77],manag:[51,52,53,54,55,56,57,59,62,64,66,69,71,83],mangag:60,mani:[74,76,77],manifest_filepath:51,mantissa:[48,72],manual:[75,76,85],manual_se:51,manylinux2014_x86_64:57,manylinux_2_17_x86_64:57,map:[1,46,59,60,62,64,66,83,86,88,90],mark:[52,60,74],markdown:51,marknodesforfallback:60,markup:[77,80],markup_process:76,markupsaf:[51,54,55,56,57],marshmallow:51,mask:[51,68],masked_fil:68,masked_sent:53,massa:79,master:[65,76,85,86,87],mat2:68,match:[48,60,85],math:80,mathemat:53,matmul:[60,68,83],matplotlib:[51,52,54,55,56,57],matric:53,matrix:65,matti:77,matur:64,mauri:[77,79],max:[47,48,54,55,56,57,58,66,68,71,74,89],max_batch_s:88,max_bound:58,max_c:89,max_dur:51,max_h:89,max_length:53,max_n:89,max_pool1d:68,max_pool2d:[56,68,82,83],max_pool3d:68,max_shap:[45,47,55,56,71,72,84],max_val:[66,68],max_valu:58,max_w:89,maxcalibr:58,maxim:55,maximu:79,maximum:[47,48,52,53,54,55,58,72,88,89],maxpool2d:[54,55],maxpool:[54,55],mayb:[55,76],mb:[52,54,55,56,57,89],md:65,me:[76,77],mean:[51,52,53,54,55,56,57,58,61,66,67,68,88],mecab:51,mechan:[51,53,66],media:[52,54,55],median:[51,53],medium:76,meet:71,mel:51,member:[46,47,48,71],memeori:2,memori:[20,21,44,45,48,51,52,54,55,56,57,60,66,71,72,83,84],memory_format:[68,71],memoryformat:[2,45],men:76,mental:76,menu:[74,76,89],menuselect:76,messag:[16,25,26,69,89],meta:80,metadata:[51,63,66,74],meth:76,method:[31,32,33,37,47,51,52,54,55,56,57,58,60,66,71,72,76,82,83,85,89,90],method_nam:[31,37,45,71,72,83,89],metric:51,metu:79,mi:79,middl:76,mig:[51,52,54,55,56],might:[53,58,60,74,85],min:[47,48,66,68,71,89],min_block_s:[45,48,61,72],min_bound:58,min_c:89,min_h:89,min_n:89,min_shap:[45,47,55,56,71,72,84],min_val:[66,68],min_valu:58,min_w:89,mind:76,mine:76,mini:[52,54,55],minim:[48,72,86,89],minimum:[47,48,55,61,69,72,89],minmax:[29,30,86],minmax_calibr:70,misbuild:74,miss:[76,83],mistun:[51,54,55,56,57],mix:57,mixin:51,mkdir:[52,54,55,85],mlm_model_t:53,mm:[51,88],mmb:76,mobilenet_v2:90,mod:[51,61,80,83,86,89],mode:[58,84,86],mode_:86,model:[51,61,63,67,69,82,83,84,86,89,90],model_math:57,model_nam:[51,58,88],model_repositori:88,model_s:51,model_state_dict:58,model_torchtrt:69,model_trt:69,modelpt:51,modern:57,modifi:[77,85],modified_state_dict:58,modul:[31,32,33,37,45,48,51,52,53,54,55,56,57,58,61,62,63,64,66,67,71,72,75,76,77,84,86,89,90,91],modular:83,module_fallback:60,module_nam:89,molesti:79,momentum:[54,55,58,68],mon:55,month:51,monthli:[51,55],morbi:79,more:[52,54,55,56,57,58,59,67,71,74,77,82,83,85,86,87,88,90],most:[53,64,85,87,88],most_likely_token_id:53,most_likely_token_ids_trt:53,mother:76,motion:76,mous:76,move:[29,44,45,52,54,55,56,57,60,63,72,83,86],mpmath:51,ms:[52,54,55,56,57,58],mse:58,msg:[26,42,51,53,69],mu:76,much:[66,74,76,86],mul:[58,68],mul_:68,multi:89,multidict:51,multipl:[63,76,77,86,88],multipli:[48,72],must:[33,47,48,53,57,60,61,66,71,72,76,77,83,85,87,89],mutil:77,my:76,myclass:76,mymodel:[61,84],mypi:57,myself:77,n01537544:52,n01739381:52,n01749939:[54,55],n01820546:[54,55],n02109961:52,n02110185:[54,55],n02481823:[52,54,55],n:[51,52,53,54,56,66,83,86,89],n_fft:51,n_mel:51,nabla:76,nam:[77,79],name:[3,4,31,33,37,44,51,52,54,55,56,57,58,61,63,66,70,72,76,77,82,83,85,88,90],named_modul:58,namespac:[42,43,44,45,50,60,67,86],narrow:[58,68],nativ:[58,64,65,83],native_funct:65,natur:[53,76],nav:[74,80],navig:74,navigation_depth:74,nbbind:[3,4,44],nbclient:[51,54,55,56,57],nbconvert:[51,54,55,56,57],nbformat:[51,54,55,56,57],nchw:[2,71,72],ncol:[52,54,55],ne:[60,68],nec:79,necessari:[42,87],need:[0,1,2,25,30,43,46,52,54,55,57,59,60,66,76,83,84,85,86,87,88],neg:68,negative_slop:68,nemo:51,nemo_1:51,nemo_asr:51,nemo_log:51,nemo_toolkit:51,nequ:[77,79],nest:[49,51,54,55,56,57,76,77],net:[52,54,55,66,76,77,83],netu:79,network:[29,30,52,54,56,57,58,66,83,86,88,91],networkx:57,neural:[52,54,57,91],new_lay:66,new_level:53,new_local_repositori:85,new_lr:58,new_siz:86,newer:[52,54,56,57],newest:51,newli:51,next:[3,4,57,58,59,63,74,76,77,86,88],nfilt:51,ngc:[51,52,53,54,55,56,57,85,88],nhwc:[2,71,89],nibh:[77,79],nice:85,nickel:76,night:77,nine:53,ninja:85,nisi:79,nisl:79,nl:[52,54,55],nlp:[29,30,53,86],nltk:51,nn:[51,52,54,55,56,58,60,65,71,72,82,83,84],no_grad:[51,52,53,54,55,56,57,58],node:[58,60,61,62,64,66,83],node_info:[66,83],noexcept:[3,4,44,86],non:[77,79],non_block:[58,68],none:[52,54,56,57,58,66,68,71,72,74,76],nonetheless:76,nonexist:76,noninteract:51,norm:68,normal:[0,1,2,46,51,52,53,54,55,56,57,58,76,82,83,86,88,91],normalized_shap:68,noskipw:44,notatemoduleforfallback:60,note:[1,46,47,53,66,71,74,76,83,85,91],notebook:[51,52,53,54,55,56,57,58,64],notic:[56,57],now:[51,52,53,54,56,57,60,64,66,76,83,85,90],np:[51,52,53,54,55,56,57,58,88],nrow:[52,54,55],nrun:[52,54,55,56,57,58],nu:76,nulla:79,nullptr:[44,45,48],num:[51,53,89],num_avg_timing_it:[45,48,72,90],num_batch:58,num_bit:58,num_calib_batch:58,num_class:58,num_epoch:58,num_it:89,num_loop:[51,53],num_min_timing_it:[45,48,72,90],num_op:89,num_work:[58,86],numba:51,number:[3,4,48,51,52,53,54,58,60,61,66,71,72,74,83,89],numel:68,numer:[51,77,89],numpi:[51,52,53,54,55,56,57,58,88],nunc:79,nvcr:[51,88],nvidia:[32,37,42,43,44,45,51,52,53,54,55,56,57,58,65,71,72,83,85,88,89,91],nvidia_convnets_processing_util:57,nvidia_deeplearningexamples_torchhub:57,nvidia_efficientnet:57,nvidia_efficientnet_b0:57,nvidia_efficientnet_b4:57,nvidia_efficientnet_widese_b0:57,nvidia_efficientnet_widese_b4:57,nvidia_resnet50:57,nvidia_resnext101_32x4d:57,nvidia_resnext:57,nvidia_se_resnext101_32x4d:57,nvidia_ssd:57,nvidia_ssd_processing_util:57,nvidia_ssdpyt_amp_200703:57,nvidia_tacotron2:57,nvidia_tts_util:57,nvidia_waveglow:57,nvinfer1:[3,4,29,30,44,45,48,66,86],nvinfer:[20,44],nwarmup:[52,54,55,56,57,58],o:[52,54,55,76,85,88],oauthlib:51,obj:68,object:[0,1,2,3,4,46,47,48,63,66,69,70,72,86,90],observ:[51,52,53,54,58],obsolet:57,obtain:[51,52,53,54,55,56,57,58,84],obvious:82,octet:[52,54,55],odio:[77,79],off:[51,52,54,55,56,57,61,63],offici:85,ofstream:[44,83],often:76,oh:77,ok:[52,54,55,76,83],okai:48,older:64,omegaconf:51,onc:[42,43,44,45,59,60,63,86,87,88],one:[53,57,58,60,66,69,71,76,82,83,88],ones:[42,52,54,56,57,61,62,64,76,83,85],onli:[1,3,4,16,30,44,46,47,56,57,60,61,64,66,69,71,76,85,86,87,89,91],onnx:[51,60],onto:[63,89],onward:[52,54,55],op:[52,53,54,55,57,58,59,60,62,64,66,71,83,87,89],op_nam:89,op_precis:[52,54,55,57],open:[52,54,55,56,57,88],opencc:51,oper:[0,1,2,3,4,31,44,45,46,48,52,54,55,56,57,58,59,60,61,62,63,64,66,67,71,72,84,86,89,91],oppos:72,opset:[62,64],opt:[47,48,51,52,53,54,55,56,57,58,71,85],opt_c:89,opt_h:89,opt_n:89,opt_shap:[45,47,55,56,71,72,84],opt_state_dict:58,opt_w:89,optim:[47,51,52,53,54,55,56,57,58,60,67,82,83,84,89],optimin:47,optimiz:[52,54,56,57,82],optimized_execut:51,optimz:88,option:[44,47,61,62,64,71,76,80,85,86,87,89,91],orchestra:76,orci:79,order:[48,57,61,66,72,83,84],org:[51,52,53,54,55,56,57,58,65,74,76,82,83,85,86],organ:77,origin:[51,53,57,58],original_nam:56,ornar:[77,79],os:[45,58],ostream:45,other:[0,1,2,45,46,52,54,55,56,57,58,59,60,63,67,68,75,76,83,84,85,87,89],otherwis:[52,53,54,85,87],our:[52,53,54,55,56,57,61,64,82,83,88],out:[31,44,51,52,53,54,55,56,58,59,60,61,62,64,66,69,72,76,83,85,88],out_dir:58,out_featur:[54,55,56],out_shap:83,out_tensor:[66,83],output0:60,output:[24,27,33,48,52,53,54,55,56,57,58,59,60,61,63,66,69,72,74,76,77,83,85,88,89],output__0:88,output_file_path:[89,91],output_pad:68,output_s:[54,55,68],output_trt:53,outself:83,outsid:76,over:[52,54,55,56,62,64,76,88],overkil:56,overrid:[3,4,29,30,44,71,86],overview:[53,65,67],own:[51,52,53,54,56,66,76,83,88],p0:55,p2:51,p8:[51,52,54,56],p:[52,54,55,68,83,88,89,91],packag:[51,52,53,54,55,56,57,58,60,83,89],pad:[51,53,54,55,58,68],padding_idx:68,padding_mod:58,page:[55,67,78,80,88],pair:[51,60,66,76,85,86],panda:51,pandocfilt:[51,54,55,56,57],pane:76,pangu:51,paper:[52,54,57],paragraph:[77,80],parallel:53,param:[70,75],param_group:58,paramet:[0,1,2,3,4,25,26,27,29,30,31,32,33,35,37,46,47,48,58,59,60,66,69,71,72,80,82,83],parameter:51,parent:[14,15,18,19,20,21],pari:53,pars:[58,76,83],parser:76,parso:[51,54,55,56,57],part:[51,61,64,74,75,76,89],parti:55,partial:[52,54,56,57,76,89],particular:56,particularli:53,partit:60,partitioninfo:61,pass:[51,53,58,59,61,62,63,64,66,69,70,82,83,86],past:76,patch:57,path:[4,13,14,15,29,30,56,57,58,70,71,82,83,85,86,88,89],path_to_torchtrt_root:85,pathspec:[51,57],pathtool:51,pathwai:82,pattern:[66,71,83],payment:75,pbtxt:88,peephole_optimz:60,pellentesqu:79,peopl:76,pep:76,per:[55,57,58],percentil:[51,53,58],perf:[51,52,54,55,56],perfom:58,perform:[29,30,52,53,54,55,56,57,86,88],performac:86,permiss:[51,52,53,54,55,56,57,58],permit:76,permut:68,persist:[51,52,54,55,56,76],pesq:51,pexpect:[51,54,55,56,57],pharetra:79,phase:[16,58,66,83],phasellu:79,phi:76,philosoph:76,phrase:76,pi:76,pick:[56,82],pick_best:57,pickler:63,pickleshar:[51,54,55,56,57],pid:[51,52,54,55,56],piec:51,pil:[52,54,55,88],pillow:[51,52,57],pin:75,pin_memori:68,pip3:85,pip:[51,52,53,54,55,56,57,58,85,88],pipelin:[89,91],piplein:83,pipreq:51,pixel_shuffl:68,pl:75,place:[47,60,76,77,78,85,86],placerat:79,plan:[64,89],plane:58,platea:79,platform:[45,52,54,56,57,64,85,88,89,91],platformdir:57,pleas:[51,52,58,76,83,85,88],plot:57,plot_result:57,plt:[52,54,55,57],pluggi:51,plugin:51,po:53,point:[71,74,75,76,83,88],pointer:[3,4,86],polish:75,pooch:51,pool:[54,55,56,57,58,91],pop:63,popular:[53,75,85],portabl:[52,54,56,57,63,72],portalock:51,portion:76,porttitor:[77,79],pos_mask:53,posit:[51,53,71,74,89],possibl:[52,53,54,56,57,76,88],post1:51,post:[29,30,48,67,83,89],posuer:[77,79],potenti:[48,79],pow:68,power:[52,54,56,57,76,83],pr:83,practic:[52,54,56,57],praesent:79,pragma:[42,43,44,45,86],pre:[33,51,52,53,54,58,60,70,72,86,87],pre_cxx11_abi:85,preced:76,precis:[48,53,55,56,57,67,71,83,84,86,89,91],precisions_str:51,pred:[52,54,55,58],pred_label:57,pred_loc:57,predict:[52,53,54,55,57],prefer:83,prefix:[27,28,42,69,76],preinstal:85,prelu:68,prepar:[51,52,53,54,56,57,88],prepare_input:57,prepare_tensor:57,preprint:86,preproc:70,preprocess:[51,52,54,55,58,86,88],preserv:[58,76,82,86],prespect:82,press:76,pretium:79,pretrain:[51,52,53,54,55,57,88,90],pretti:83,prev_next_buttons_loc:74,prevent:[48,89],previou:[53,74],previous:[30,33,83],prim:[59,60,63,68,82,83],prim_devic:68,primal:76,primari:53,primarili:[64,83],print:[16,31,44,51,52,53,54,55,56,57,58,69,71,72,76,83,88,90],print_funct:51,printout:53,printstat:[51,53],priorit:85,privat:[3,4,44,45,86],prob:[52,54,55],probabl:[52,53,54,55,57],probablil:[52,54,55],problem:[53,76],problemat:76,proce:[52,54,55,88],proceed:88,process:[51,52,53,54,55,56,57,58,61,75,76,82,86,88,89,90],prod:68,produc:[47,59,63,66,76,83],product:[48,52,54,56,57],profil:[47,56],program:[18,19,20,21,30,50,55,56,57,62,63,64,67,82,89],programm:76,progress:77,proin:79,project:[75,80],prometheu:[51,54,55,56,57],promis:51,prompt:[51,54,55,56,57],properli:85,properti:[51,53,74],propog:60,prose:76,protobuf:51,provid:[3,4,48,51,52,53,54,55,61,63,66,71,72,76,83,84,85,86,87,88,89,90],providi:[62,64],provok:76,psutil:[51,55],pt:[53,57,58,83,88,89],pth:[54,57,58],ptq:[3,4,15,18,38,49,50,67,71,72,89],ptq_calibr:[3,4,45,48,86],ptqtemplat:49,ptyprocess:[51,54,55,56,57],publish:88,pull:[85,88],purchas:75,pure:[31,51,55,57],purpos:[55,57,85,88],puru:79,push:63,push_back:[44,61],put:76,pwd:88,pwr:[51,52,54,55,56],py2:[54,56,57],py3:[51,52,53,54,56,57,88],py:[51,52,57,58,60,64,74,76,81,82,83,85,86],pyannot:51,pyasn1:51,pybind11:51,pybtex:51,pycpars:[51,54,55,56,57],pycr:51,pydeprec:51,pydub:51,pygment:[51,54,55,56,57],pyindex:88,pypa:[51,52,53,54,55,56,57],pypars:[51,53,54,55,56,57],pypi:[51,52,53,54,55,56,57,58,85],pypinyin:51,pyplot:[52,54,55,57],pyrsist:[51,54,55,56,57],pysock:51,pystoi:51,pytest:51,python3:[51,52,53,54,55,56,57,58,60,83,85],python:[51,52,53,54,55,56,57,58,61,64,71,72,76,77,83,87,88,89,90,91],python_api:65,pythonhost:[54,55,56,57,58],pytorch:[47,48,51,52,53,54,55,57,58,60,61,62,63,64,66,70,71,72,82,83,84,85,86,87,88,89],pytorch_libtorch:88,pytorch_lightn:51,pytorch_quant:[57,58],pytorch_sphinx_them:[74,81],pytorch_vision_v0:55,pytz:51,pywavelet:57,pyyaml:[51,53],pyzmq:[51,54,55,56,57],qat:58,qat_model:58,qthelp:51,qualiti:[52,54,57],quant:58,quant_dim:58,quant_input:58,quant_max:68,quant_min:68,quant_modul:58,quant_nn:58,quant_weight:58,quantconv2d:58,quantdescriptor:58,quantiz:[29,30,57,67,83,89],quantizatiom:48,quantizelay:58,quantlinear:58,quantoz:58,quantpool:58,quartznet:51,question:83,qui:[77,79],quick:58,quickli:[52,54,83,86,89],quisqu:79,quit:[55,66,83],quot:77,r:[57,76],rais:60,raiseexcept:60,rand:83,randn:[51,52,54,55,56,57,58,61,71,72,83,90],random:51,randomcrop:58,randomhorizontalflip:58,rang:[47,48,51,52,53,54,55,56,57,58,71,89],rank:74,rapidfuzz:51,rate:58,rather:60,raw:[57,74],re:[51,76],read:[3,4,29,30,44,51,55,74,76,86],read_calibration_cach:70,readcalibrationcach:[3,4,44],reader:76,readi:[51,55],readm:[51,52,53,54,56,57],realiz:63,realli:66,reason:[0,57,82],reattribut:77,recalibr:30,recip:86,recipi:57,reciproc:68,recognit:[51,54,58,86],recomend:[29,30],recommend:[29,30,51,52,53,54,55,56,57,58,76,83,85,88],recompil:57,record:[56,58,59,82],rect:57,rectangl:57,recurs:59,recursivescriptmodul:56,redistribut:77,reduc:[52,54,56,57,58,60,62,64,86],ref:76,refer:[47,58,62,64,75,80,83,84,86,88],referenc:[57,85],refit:[45,48,72,90],reflect:45,reflection_pad1d:68,reflection_pad2d:68,regard:[76,85],regardless:77,regex:[51,53],regist:[33,63,66,72],registernodeconversionpattern:[66,83],registri:[59,83],regular:58,reinterpret_cast:44,rel:89,relat:[46,76],relationship:49,releas:[51,53,76],relu:[54,55,56,61,68,82,83],relu_:68,remain:[52,53,54,56,57,60,86],remov:[51,52,54,56,57,58,74],remove_contigu:60,remove_dropout:60,remove_to:60,render:74,rent:77,repack:63,repeat:[68,89],replac:[53,57,60],replication_pad1d:68,replication_pad2d:68,replication_pad3d:68,report:[23,44],reportable_log_level:69,repositori:[52,54,57,64,74,81,88],repres:[47,48,58,66,69,76],represent:[52,53,54,56,57,60,66,82],request:[51,52,53,54,55,71,83,88],requir:[30,48,51,52,53,54,55,56,57,58,59,60,69,71,72,74,83,86,87,88,89],require_full_compil:[45,48,52,54,56,57,72],requires_grad:68,resampi:51,research:[52,54,56,57],reserv:[42,43,44,45,51,52,53,54,55,56,57,58],reset:44,reshap:[68,88],residu:54,resiz:[52,54,55,88],resnet50:[54,55,57,88],resnet50_model:[54,55],resnet:[55,57,63,88],resnet_trt:63,resolv:[52,54,55,59,60,62,64],resolve_data_config:52,resourc:[51,54,55,56,57,59,86],respons:[30,52,54,55,58,63,76],respositori:53,rest:[76,77],restor:51,restrict:[48,72],restructuredtext:[76,77],result:[51,52,53,54,55,56,58,59,60,69,72,74,82,84,88],results_per_input:57,ret:60,rethink:52,return_tensor:53,reus:[60,86],revert:74,revis:[76,77],revisit:76,rfc:76,rgb:[52,54],rho_:76,rhoncu:79,right:[42,43,44,45,51,52,53,54,55,56,57,58,60,64,66,76],risu:79,rm:88,rn50_preprocess:[54,55,88],role:76,roll:68,roman:77,room:76,root:[42,43,44,45,51,52,53,54,55,56,57,58,74,85,86],roughli:61,round:[48,58,72],round_:58,rounding_mod:68,row:77,rsa:51,rst:[74,76],rsub:68,rtol:89,ruamel:51,rule:[72,85],ruler:76,run:[1,37,46,48,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,67,71,72,76,82,83,84,85,86,87,88,89,90,91],runner:51,running_loss:58,running_mean:68,running_var:68,runtim:[51,52,54,55,56,57,67,83],rutrum:[77,79],s3:[52,54,55],s3transfer:51,s:[47,48,57,58,61,63,66,67,71,74,76,77,82,83,84,86,88],sacrebleu:51,sacremos:[51,53],safe:[66,72],safe_dla:71,safe_gpu:71,safeti:[48,71,89],sage:76,sagitti:[77,79],sai:[55,77],said:76,same:[52,54,55,57,63,74,76,82,83,85,88,90],sampl:[51,52,54,76,86,88],sample_r:51,sapien:79,satisfi:[51,52,53,54,55,56,57,61],save:[30,44,51,52,54,55,56,57,58,63,71,72,83,84,87,88,89],save_checkpoint:58,save_restore_connector:51,saw:83,scalar:[66,68],scalaropt_dim:68,scalartyp:[0,45,68],scale:[52,58,68,86],scale_factor:68,scale_grad_by_freq:68,scales_d:68,scales_h:68,scales_w:68,scelerisqu:79,schedul:[58,71,88],schema:[66,83],scientist:76,scikit:[51,57],scikit_imag:57,scipi:[51,57],scope:60,score:[52,54,55,57],scratch:30,scratch_spac:88,screen:74,script:[31,53,57,58,60,61,71,72,82,83,84,90],script_model:[56,82,90],scriptclass:72,scripted_model:91,scriptmodul:[71,72,83],scroll:[74,78],sdk:[51,52,54,56,57,65],se:51,seamlessli:[55,67],search:[53,67,74],second:[52,53,55,60,76],secondli:88,section:[55,58,74,76,77,78,80,83,86,88],secur:[51,85],sed:[77,79],see:[31,51,52,53,54,55,56,57,58,60,63,71,72,76,82,83,85],seen:[76,77],segment:[51,61],segments_tensor:53,select:[17,29,30,37,48,52,54,56,57,63,68,71,72,75,78,85,86,89],self:[51,53,56,58,60,63,66,68,70,82,83,91],self_1:[63,83],self_int:68,sell:77,seller:75,seller_id:75,sem:79,semant:76,semper:79,send2trash:[51,54,55,56,57],send:88,senectu:79,sens:[76,83],sent:[52,54,55],sentenc:[53,76],sentencepiec:51,sentencepiecetoken:51,sentinel:[0,2],sentri:51,separ:[52,54,56,57,61,62,64],seq_relationship:53,sequenc:[51,53,76],sequenti:[54,55],serial:[33,37,62,64,71,72,83,89],seriali:72,serializ:[63,82],serialized_engin:72,serializinghtml:51,seril:63,serv:[63,67,89],server:51,servic:76,session:76,session_nam:76,set:[3,4,16,21,25,27,30,32,35,37,45,46,47,48,52,54,56,57,58,59,60,61,62,63,64,67,69,71,72,74,78,81,82,83,84,85,86,91],set_data_from_numpi:88,set_devic:[21,38,45,49,71],set_is_colored_output_on:[18,39,42,49,69],set_logging_prefix:[18,39,42,49,69],set_reportable_log_level:[18,39,42,49,53,58,69],set_typecheck_en:51,setalpha:66,setbeta:66,setnam:[66,83],setproctitl:51,setreshapedimens:83,setup:[43,51,58,86,88],setup_multiple_test_data:51,setup_multiple_validation_data:51,setup_test_data:51,setup_training_data:51,setup_validation_data:51,setuptool:[51,54,55,56,57],sever:[16,26,53,69],sgd:58,sh:85,sha256:85,shape:[45,47,48,51,52,53,54,56,57,58,61,66,68,71,72,84,88,89,91],shape_mod:71,share:85,shell_command:76,shellingham:51,shift:[68,76,85],ship:[58,83,87],shorthand:76,shortuuid:51,should:[0,3,4,30,45,48,52,55,58,59,60,61,62,64,66,67,69,71,74,76,79,86,88,89],show:[57,74,76],showcas:[52,54,55],shown:[53,74,76,83],shuffl:[51,58,83,86],shutterstock_780480850:[52,54,55],siberian:[52,54,55],siberian_huski:[54,55],side:[60,74,83],sidebar:[74,80],sigmoid:68,sigmoid_:68,sign:88,signatur:72,signifi:[47,60],signific:[57,58,76],significantli:[60,74],similar:[57,66,83,87,90],simonyan:86,simpil:86,simpl:[51,52,53,54,56,57,58,76,77,82,88],simplejson:51,simplest:[53,88],simpli:[55,56,60],simplic:[52,54,57],simplifi:59,simul:58,sin:[68,76],sinc:[53,56,60,76,82,83,86],sing:76,singl:[47,48,53,56,60,71,76,82,83,86,89],singular:66,sinh:68,sink:76,sit:[77,79],site:[51,52,53,54,55,56,57,58,60,76,83,85],six:[51,53,54,55,56,57,76],sixth:77,size:[3,4,44,47,48,51,52,53,54,55,56,57,58,60,61,68,71,72,74,83,86,89,91],size_t:[3,4,44,86],skip:89,slash:74,slice:68,slither:[52,54,55],sm:63,sm_output:[52,54,55],small:[58,60,88],smaller:51,smallest:53,smi:[51,52,54,55,56],smmap:51,snake:[52,54,55],snowballstemm:51,so:[0,44,52,54,55,56,58,59,60,63,64,66,67,75,76,77,83,85,86,89],sodal:79,softmax:[52,54,55,57,58,60,68],softwar:[51,52,53,54,55,56,57,58,76],sole:86,sollicitudin:79,solv:88,some:[52,53,54,59,60,62,63,64,66,75,76,83,86],some_funct:76,someth:[43,60,76,88],someurl:76,sort:[66,68,90],sortedcontain:51,soundfil:51,soupsiev:[51,55],sourc:[42,43,44,45,52,54,57,64,69,70,71,72],sourceforg:[76,77],sox:51,space:[76,77,86],spaces_and_linebreak:76,span:77,spars:[68,89],sparse_weight:[45,48,72],sparsiti:[48,72,89],speak:53,speaker:53,spec:[47,48,52,54,56,57,69,71,72,89,90],specif:[32,48,51,52,53,54,55,56,57,58,60,62,64,71,72,76],specifi:[3,4,48,52,53,54,55,56,57,58,66,67,69,71,72,74,76,84,88,89,90],specifii:71,speech:51,speed:[51,52,53,54,55,57],speed_m:[51,53],speed_mean:[51,53],speedup:[51,52,53,54],sphinx:[51,74,75,76,77,81],sphinx_rtd_them:[76,77],sphinxcontrib:51,spin:88,spirit:76,split:[53,68],split_siz:68,split_with_s:68,sqrt:68,squeez:[51,68],sr:51,src:[63,65,68],ss:44,ssd300:57,ssd300_trt:63,ssd:63,ssd_300_trace:57,ssd_pyt_ckpt_amp:57,ssd_trace:89,ssd_trt:89,sstream:[20,44],stabl:[58,65,74],stack:[51,55,57,58,63,68,86],stage:59,stand:[63,76],standalon:76,standard:[52,53,54,55,56,57,63,67,76,87,89,90],stapl:77,start:[53,55,57,58,59,61,68,77,85,90],start_dim:[68,83],start_step:68,start_tim:[51,52,53,54,55,56,57,58],startswith:58,stat:58,state:[51,52,53,54,58,59,66,83],state_dict:58,statement:[60,76],static_cast:44,statist:[53,58],statu:[44,77],std:[3,4,22,26,28,29,30,31,33,34,37,42,44,45,47,48,51,52,53,54,55,61,83,86,88,91],std_dev:[51,53],stderr:58,stdout:[36,69,71],steamlin:86,step:[51,52,53,54,55,56,57,58,67,68,86],stft:51,stick:74,sticki:[74,80],sticky_navig:[74,78],still:[44,57,61,86],stitch:[56,61,83],stop:83,storag:86,store:[2,4,59,63,66,82,83],str:[19,43,44,49,52,54,55,68,69,71,72],straight:66,strang:76,strategi:[53,71],stream:[52,54,55],street:77,strict:87,stride:[54,55,56,57,58,68],string:[3,4,18,20,21,22,26,28,29,30,31,33,34,37,42,44,45,48,61,63,66,71,74,83,86],stringstream:44,strip_prefix:85,strong:[52,54,56,57,76],strongli:76,struct:[1,21,38,41,45,86],structur:[30,46,48,52,54,56,57,61,64,66,74,76,80,82,88],structuredtext:76,stt_en_citrinet_256:51,stt_en_citrinet_256_bs128_torch:51,stt_en_citrinet_256_bs1_torch:51,stt_en_citrinet_256_bs32_torch:51,stt_en_citrinet_256_bs8_torch:51,stub:77,stuff:76,style:[42,43,44,45,74,76,77],style_external_link:74,sub:[68,76,82],sub_:68,subdirectori:50,subexpress:60,subgraph:[48,59,60,66,83,89],subject:64,submenu:80,submodul:[56,82],subplot:[52,54,55,57],subscript:76,subsect:76,subset:[58,86],substitut:76,subtitl:76,subtre:81,subword:51,successfulli:[51,52,54,56,57],sudo:85,suffic:60,suggest:88,suit:[55,67],sum:[48,58,68,72],summari:53,summarywrit:58,superscript:76,suppli:76,support:[0,1,2,27,31,46,47,48,52,54,55,56,57,58,61,65,67,71,72,74,75,82,83,85,88,89,91],sure:[83,84,85,88,91],suscipit:[77,79],suspendiss:79,swap:51,sy:58,symbol:[33,72,76,85,87],symlink:81,sympi:51,synchron:[51,52,53,54,55,56,57,58],system:[51,52,53,54,55,56,57,59,66,67,72,85],t1:68,t2:68,t:[0,1,2,45,46,55,56,58,60,66,68,74,76,77,82,83,85,86,88],t_:76,tabl:[80,85],tabul:51,tag:[76,88],take:[31,32,33,37,51,52,54,56,57,59,62,63,64,66,71,72,74,76,83,86,90],taken:[52,54,57,76],talk:67,tan:68,tanh:68,tanh_:68,tar:[76,85,86],tarbal:[83,86],target:[1,33,45,46,47,48,52,54,55,56,57,63,64,67,71,72,84,86,89,90,91],targets_:86,tarred_audio_filepath:51,task:[29,30,51,53,86],techinqu:83,techniqu:[58,86],tell:[60,61,62,63,64,66,76],tellu:79,tem:89,temp:[51,52,54,55,56],templat:[20,40,44,45,49,74,83],tempu:79,tensor:[2,33,44,45,47,48,51,52,53,54,55,56,57,58,59,60,61,63,66,68,71,72,82,83,86],tensor_mod:68,tensor_qu:58,tensor_quant:58,tensor_scalar:68,tensor_tensor:68,tensorboard:[51,58],tensorcontain:66,tensorformat:[21,38,45,47,49,71],tensorformatenum:49,tensorlist:[61,66],tensorquant:58,tensorrt:[0,1,3,4,29,30,31,32,33,36,37,44,45,46,47,48,53,59,60,61,62,64,66,70,71,72,82,86,89],tensorrtcompilespec:[72,90],teo:89,term:[55,71,76,77,86],termin:[27,83,89],terminado:[51,54,55,56,57],test:[51,52,53,54,55,56,57,58,64,76,77,86,88,89],test_acc:58,test_loss:58,test_pr:58,test_prob:58,test_ptq_dataloader_calibr:86,test_ptq_trt_calibr:86,test_py_modul:[76,80],testing_dataload:[58,86],testing_dataset:[58,86],testpath:[51,54,56,57],text:[51,53,57,69,77,79],tf32:[48,89],tgz:85,than:[51,53,55,60,67,75,76,87],thats:[59,86],the_model_repositori:88,thei:[46,53,57,58,59,60,63,66,71,74,76,85,89],them:[51,52,53,54,56,57,60,61,63,74,83,85],theori:[59,76],therebi:63,therefor:[30,51,52,54,56,57,63,76,83],theres:87,therfor:87,theta:76,thi:[0,1,2,29,30,42,43,44,45,46,47,48,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,71,72,74,75,76,78,79,82,83,85,86,87,88,89,90],thicker:76,thin:76,thing1:76,thing2:76,thing3:76,thing:[56,76,85],think:[66,76],third:[55,77],third_parti:[64,85],those:[53,59,76],though:[57,64,66,82,83,89],thought:76,threadpoolctl:51,three:[47,55,62,64,71,76,77,88],threshold:89,through:[47,51,52,53,54,55,56,57,59,60,61,63,67,69,70,76,83],throughout:[52,54],throughput:52,thrown:[48,72],thu:[51,56,76],tifffil:57,time:[48,51,52,53,54,55,56,57,58,59,60,62,63,64,66,72,74,76,83,86,89],time_99th:[51,53],time_m:[51,53],time_mean:[51,53],time_std:[51,53],timegraph:[51,53],timeit:[51,53],timeout:51,timm:[52,54],tincidunt:79,tini:86,tinycss2:55,titan:[51,52,54,56,57],titl:[52,54,55],titles_onli:74,tmp:83,toctre:74,tocustomclass:66,todim:83,todo:74,togeth:[56,59,66,83],toilet:[52,54,55],token:[51,53],token_type_id:53,tokens_tensor:53,toler:89,toml:51,tomli:57,too:[74,76,77,85],took:53,tool:[52,53,54,56,57,66,83],toolchain:[64,85],toolkit:[51,54,55,56,57,58],top:[57,64,74,78],topk:68,topolog:53,torch:[0,1,2,4,20,29,30,31,32,33,36,37,44,45,46,47,48,53,59,60,61,62,63,64,66,71,72,82,85,86,89,91],torch_executed_modul:[45,48,61,72],torch_executed_op:[45,48,61,72],torch_scirpt_modul:82,torch_script_modul:83,torch_tensorrt:[0,1,2,3,4,14,16,17,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,61,67,83,84,86,87,88,89,90,91],torch_tensorrt_major_vers:[19,43,49],torch_tensorrt_minor_vers:[19,43,49],torch_tensorrt_patch_vers:[19,43,49],torch_tensorrt_vers:[19,43,49],torch_tensorrtfil:49,torch_tensorrtnamespac:49,torchbind:63,torchhub:[57,88],torchmetr:51,torchscript:[19,21,38,43,45,48,49,51,52,53,54,55,57,58,62,63,64,71,72,84,89,90,91],torchscriptstruct:49,torchtrt:[43,51,61],torchtrt_api:[19,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,42,43,44,45,49],torchtrt_check:66,torchtrt_hidden:[19,43,49],torchtrt_runtime_exampl:87,torchtrt_unus:66,torchtrtc:[67,91],torchvis:[51,52,54,55,58,63,86,88,90],tornado:[51,54,55,56,57],toronto:86,tortor:79,total:58,totensor:[52,54,55,58,86,88],tovec:83,toward:86,tqdm:[51,53,58],trace:[51,53,57,58,61,72,82,83,84],traced_mlm_model:53,traced_model:[56,57,82],tracerwarn:58,track:[66,86],track_running_stat:[54,55],trade:57,tradit:[47,72,86],traget:32,trail:74,train:[29,30,48,51,52,53,54,57,67,68,83,84,89],trainabl:60,trained_vgg16_qat:58,trainer:51,training_dataload:58,training_dataset:58,traitlet:[51,54,55,56,57],transcrib:51,transfer:75,transform:[51,52,54,55,56,57,58,83,86,88],transformed_img:88,transforms_factori:52,translat:[57,83],transmit:76,transpos:68,trash:76,travers:[62,64],treat:[58,89],tree:[42,43,44,45,51,74,86,87],trigger:[56,83],trim:86,trim_sil:51,tristiqu:79,triton:67,triton_to_np_dtyp:88,tritoncli:88,tritonserv:88,trt:[0,1,3,4,46,47,51,59,60,63,66,68,83],trt_lenet_script:83,trt_mod:[58,61,83,86,91],trt_model:[53,57,61,88,90],trt_model_fp16:[52,53,54],trt_model_fp32:[52,54],trt_model_with_d:55,trt_model_without_d:55,trt_script_modul:56,trt_ts_modul:[51,56,61,84],trtorch:51,truck:58,truncat:[48,72,89],truncate_long_and_doubl:[45,48,51,53,72],trust:[54,55,56,57,58],ts:[43,51,56,61,67,71,82,83,84,89,90],ts_model:[61,83],tt:76,tue:[54,77],tune:[51,52,54,56,57,58],tupl:[63,71,72],tupleconstruct:[60,63],tupleunpack:60,turn:51,turpi:79,tutori:[52,54,55,82,86],two:[51,56,57,60,66,76,77,81,82,85,86,88,89],type:[0,1,2,29,47,48,49,51,52,53,54,55,56,57,58,59,63,66,69,71,72,76,83,86,89],type_fp32:88,typecheck:51,typenam:[3,4,29,30,44],typer:51,typic:[59,66,88],typing_extens:52,ubuntu:[51,85],ugli:76,ui:75,uint64_t:[45,48],ultric:79,un:[52,54,86],unabl:[66,83],unbind:68,unbroken:76,uncas:53,unchang:53,uncom:85,uncorr:[51,52,54,55,56],undefin:57,under:[42,43,44,45,51,52,53,54,55,56,57,58,64,76,84],underli:[0,1,2,46,66],understand:[52,54],unidecod:51,union:[66,71,72,83],uniqu:4,unique_ptr:[4,29],unit:[53,56],univers:76,unknown:71,unless:[51,52,53,54,55,56,57,58],unlik:[55,67,85,90],unlimit:74,unmask:53,unmasked_sent:53,unmasked_sentences_trt:53,unmasked_token:53,unmasked_tokens_trt:53,unpack_addmm:60,unpack_log_softmax:60,unqiue_ptr:4,unreferenc:76,unrestrict:76,unsign:58,unsqueez:[52,54,55,68],unstabl:64,unsupport:[31,48],unsur:66,untest:64,until:[55,59,64,66,85],unwrap:66,unwraptodoubl:66,unwraptoint:83,unzip:85,up:[51,52,53,54,56,57,58,59,60,62,63,64,76,82],updat:[51,55,58],upgrad:51,upload:[52,54,55,88],upon:74,upper:[58,77],upsample_bilinear2d:68,upsample_linear1d:68,upsample_nearest1d:68,upsample_nearest2d:68,upsample_nearest3d:68,upsample_trilinear3d:68,upscale_factor:68,upstream:83,uri:[57,76],url:[74,85,88],urllib3:[51,53],urna:79,us:[0,1,2,3,4,29,30,32,35,37,43,44,45,46,47,48,51,52,53,54,56,57,59,61,63,64,66,67,69,70,71,72,74,75,76,77,82,86,87,88,89,91],usag:[51,52,54,55,56,70,76,83],use_amp:51,use_cach:[3,4,29,44,70,86],use_cache_:44,use_fb_fake_qu:58,use_input_stat:68,use_start_end_token:51,use_subset:86,usecas:85,user:[42,47,48,51,52,53,54,55,56,57,61,62,63,64,76,77,83,85,86,88],userguid:58,userwarn:[51,52,57],using_int:[68,83],usr:85,usual:[57,58,74],ut:79,utf:[76,77],util:[52,54,56,58,66,72,83,86,88],v0:[54,55,73,88],v1:51,v2:[29,30,57],v8:85,v:[51,52,53,54,56,57,77,88,89],val2017:57,val:[57,58],valid:[1,46,51,56,57,58,66],valu:[0,1,2,16,17,45,46,47,53,56,58,59,63,66,68,69,70,71,74,83],value_tensor_map:[59,66],vari:[52,53,54,55],variabl:[47,71],variant:[51,87],varient:60,varieti:88,variou:[51,91],variu:79,vcs_pageview_mod:74,vec:68,vector:[20,21,44,45,47,48,61,63,83,86,91],vehicula:79,vel:79,velit:79,venenati:79,venv:[51,52,53,54,55,56,57],verbios:89,verbos:[77,89],veri:[58,77,78,86,88,90],verifi:[53,58,61],version:[34,36,51,52,53,54,55,56,57,58,64,74,77,85,88],vertic:[74,76],vestibulum:[77,79],vgg16:[58,86],vgg16_base_ckpt:58,vgg16_qat_ckpt:58,vgg:[57,58,86],vi:76,via:[51,55,57,67,71,72,74,80,84,86,87],view:[68,74],vine_snak:52,virtual:[51,52,53,54,55,56,57,86],vision:[52,53,54,55,88],visit:[52,54,55,57],visitor:74,visual:55,vita:[77,79],vivamu:79,viverra:79,vm:77,volatil:[51,52,54,55,56],volta:[52,54,56,57],volutpat:79,vs:[0,1,2,46,60,72,90],vulput:79,w1109:58,w:[51,52,54,57,89],w_hh:68,w_ih:68,wa:[51,52,53,54,56,57,60,63,76,83],wai:[52,54,58,82,83,85,86,89],walk:[51,52,53,54,56,57],walkthrough:55,wandb:51,want:[42,52,54,56,57,61,82,83,86,88,90],warm:[51,52,53,54,55,56,57,58],warn:[16,44,51,52,53,54,55,56,57,58,66,69,89],warranti:[51,52,53,54,55,56,57,58],wash:76,wcwidth:[51,54,55,56,57],we:[42,44,51,52,53,54,55,56,57,58,59,60,62,63,64,66,74,76,82,83,86,88],weak:76,web:76,webdataset:51,webencod:[51,54,55,56,57],websit:85,weight:[47,48,53,58,59,68,72,76,83,89],weight_decai:58,welcom:83,welecom:[52,54],well:[48,52,53,54,56,57,69,76,83,85,86],were:[53,57,83],werkzeug:51,wget:[51,52,54,55,88],what:[4,57,60,76,82,83],whatev:63,wheel:[51,85],when:[27,44,45,46,52,53,54,56,57,58,59,60,62,63,64,66,69,71,72,74,76,78,82,83,85,86,89],where:[51,52,54,56,59,60,66,72,77,83,86],whether:[4,71,75,86,89],which:[1,2,30,32,37,46,51,52,53,54,55,56,57,58,59,60,61,62,63,64,66,70,72,74,76,77,82,83,84,85,86,87,88,90],white:[57,76],whitespac:76,whl:[52,54,56,57,85],who:76,whole:[52,54,56,57],whose:60,why:76,wide:[55,80],widespread:53,widget:[51,54,55,56,57],widgetsnbextens:[51,54,55,56,57],width:[52,54,55,76],window:76,window_nam:76,wish:77,wit:51,within:[51,52,53,54,55,56,57,62,64,72,74,76],without:[51,52,53,54,56,57,58,66,74,76,83,86],wl:87,won:55,wooden:76,word:[51,53,76],wordninja:51,work:[44,56,60,64,66,76,77,86],worker:86,workflow:[58,90],workspac:[48,72,85,86,89,91],workspace_s:[45,48,51,52,53,54,55,57,72,86,89,91],world:[52,54,56,57,76],would:[66,83,85,87,88,89,90],wp:[52,54,55,88],wrap:[58,62,63,64,76,79,83,90],wrapper:66,wrapt:51,write:[3,4,29,30,44,51,52,53,54,55,56,57,58,59,67,76,83,86,88],write_calibration_cach:70,writecalibrationcach:[3,4,44],writer:58,written:[52,54],wrote:76,www:[51,52,53,54,55,56,57,58,74,76,83,85,86,88],x64:85,x86:87,x86_64:[64,85],x9:60,x:[5,10,33,43,52,54,56,57,58,60,72,77,82,83,85],x_0:76,x_1:76,x_2:76,x_3:76,x_4:76,x_:76,xavier:[45,52,54,56,57,91],xstr:[19,43,49],xx:88,y:[33,51,57,72,77],yahoo:77,yaml:[51,65],yarg:51,yarl:51,year:51,yield:53,you:[0,1,2,29,30,46,47,48,51,52,53,54,55,56,57,58,59,60,61,63,64,66,67,71,72,74,76,77,78,82,83,84,85,86,87,88,89,90],your:[51,52,53,54,55,56,57,58,66,67,74,76,77,81,82,83,84,85,87,90],yourself:[52,53,54,83],youtokentom:51,yy:[51,88],z:77,zero_grad:58,zero_point:68,zeroth:55,zip:[54,57,63,85],zipp:[51,54,55,56,57],zisserman:86},titles:["Class DataType","Class Device::DeviceType","Class TensorFormat","Template Class Int8CacheCalibrator","Template Class Int8Calibrator","Define STR","Define TORCH_TENSORRT_PATCH_VERSION","Define TORCH_TENSORRT_MAJOR_VERSION","Define TORCH_TENSORRT_MINOR_VERSION","Define TORCHTRT_API","Define XSTR","Define TORCHTRT_HIDDEN","Define TORCH_TENSORRT_VERSION","Directory cpp","Directory include","Directory torch_tensorrt","Enum Level","Enum EngineCapability","File logging.h","File macros.h","File ptq.h","File torch_tensorrt.h","Function torch_tensorrt::logging::get_logging_prefix","Function torch_tensorrt::logging::get_reportable_log_level","Function torch_tensorrt::logging::get_is_colored_output_on","Function torch_tensorrt::logging::set_reportable_log_level","Function torch_tensorrt::logging::log","Function torch_tensorrt::logging::set_is_colored_output_on","Function torch_tensorrt::logging::set_logging_prefix","Template Function torch_tensorrt::ptq::make_int8_calibrator","Template Function torch_tensorrt::ptq::make_int8_cache_calibrator","Function torch_tensorrt::torchscript::check_method_operator_support","Function torch_tensorrt::torchscript::compile","Function torch_tensorrt::torchscript::embed_engine_in_new_module","Function torch_tensorrt::get_build_info","Function torch_tensorrt::set_device","Function torch_tensorrt::dump_build_info","Function torch_tensorrt::torchscript::convert_method_to_trt_engine","Namespace torch_tensorrt","Namespace torch_tensorrt::logging","Namespace torch_tensorrt::ptq","Namespace torch_tensorrt::torchscript","Program Listing for File logging.h","Program Listing for File macros.h","Program Listing for File ptq.h","Program Listing for File torch_tensorrt.h","Struct Device","Struct Input","Struct CompileSpec","Torch-TensorRT C++ API","Full API","Torch-TensorRT Getting Started - CitriNet","Torch-TensorRT Getting Started - EfficientNet-B0","Masked Language Modeling (MLM) with Hugging Face BERT Transformer","Torch-TensorRT Getting Started - ResNet 50","Torch-TensorRT - Using Dynamic Shapes","Torch-TensorRT Getting Started - LeNet","Object Detection with Torch-TensorRT (SSD)","Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT","Conversion Phase","Lowering Phase","Partitioning Phase","Compiler Phases","Runtime Phase","System Overview","Useful Links for Torch-TensorRT Development","Writing Converters","Torch-TensorRT","Operators Supported","torch_tensorrt.logging","torch_tensorrt.ptq","torch_tensorrt","torch_tensorrt.ts","Changelog","Configuration","5. :mod:`test_py_module`","3. Paragraph Level Markup","4. Lists & Tables","1. Long Sticky Nav","1. Structural Elements","<no title>","Installation","Creating a TorchScript Module","Getting Started with C++","Using Torch-TensorRT in Python","Installation","Post Training Quantization (PTQ)","Deploying Torch-TensorRT Programs","Serving a Torch-TensorRT model with Triton","torchtrtc","Using Torch-TensorRT Directly From PyTorch","DLA"],titleterms:{"1":[78,88],"10":78,"11":78,"12":78,"13":78,"14":78,"15":78,"16":78,"17":78,"18":78,"19":78,"2":[78,79,88],"20":78,"3":[78,88],"4":78,"5":78,"50":54,"6":[57,78],"7":[57,78],"8":78,"9":78,"class":[0,1,2,3,4,20,21,38,40,41,49,70,71],"enum":[16,17,18,21,38,39,49,70,71],"function":[18,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,49,55,65,71,72],"long":[78,80],A:76,And:76,But:77,By:[18,19],Or:60,The:[76,83],To:60,aarch64:85,abi:[63,85],addmm:60,admonit:76,advic:66,ahead:67,an:80,api:[49,50,65,67,85],applic:86,arg:[66,75],automat:61,avail:65,awar:58,b0:52,background:[63,66],base:[3,4,74],benchmark:[51,55,57,58],bert:53,binari:85,block:76,branch:60,build:[55,74,85,88],bullet:77,c:[49,65,67,83,85,86],can:77,caption:[77,80],center:76,ch:76,changelog:73,check_method_operator_support:31,choos:85,citat:[76,86],citrinet:51,cli:85,client:88,code:[60,76],compil:[32,62,64,67,83,85],compilespec:48,compound:76,conclus:[56,57],configur:74,construct:63,content:[18,19,20,21,38,39,40,41,51,52,53,54,56,57,74,75,76,77,78,79],context:[66,74],contigu:60,contract:66,contributor:67,convers:[59,62,64,66],convert:[59,66,68,83],convert_method_to_trt_engin:37,cpp:[13,18,19,20,21,61],creat:[82,86],creativ:76,cudnn:85,current:68,custom:83,cxx11:85,data:[55,75],datatyp:0,dead:60,debug:85,deeper:77,defin:[5,6,7,8,9,10,11,12,19,49],definit:[18,19,20,21,77],demo:80,depend:85,deploi:[58,87],descript:[52,54,57],deseri:63,detail:57,detect:57,detector:57,develop:65,devic:[1,46],devicetyp:1,dimens:65,direct:76,directli:90,directori:[13,14,15,50],disk:82,distribut:85,dla:91,doctest:76,documen:67,document:[0,1,2,3,4,5,6,7,8,9,10,11,12,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,65,67,79,80],down:77,download:[55,76,81],dr:55,dropout:60,dump_build_info:36,dynam:55,easier:65,efficientnet:52,element:79,elimin:60,eliminatecommonsubexpress:60,embed_engine_in_new_modul:33,emphas:76,engin:63,enginecap:17,enumer:77,envior:85,evalu:[59,68],exampl:[76,78],execept:60,executor:63,expect:65,explan:55,face:53,fallback:[60,61],field:77,figur:76,file:[15,18,19,20,21,42,43,44,45,49,50],flatten:60,footnot:76,format:63,fp16:[51,52,54],fp32:[51,52,54],freez:60,from:[55,85,90],full:[49,50],fuse:60,gaurd:60,gener:75,get:[51,52,54,55,56,67,83],get_build_info:34,get_is_colored_output_on:24,get_logging_prefix:22,get_reportable_log_level:23,giant:77,git:81,glossari:76,gpu:67,graph:[60,63],grid:77,guarante:66,h:[18,19,20,21,42,43,44,45,61],half:[51,52,54],have:77,hierarchi:49,hlist:77,hole:77,hood:83,how:[74,86],html:74,hub:55,hug:53,ien:76,imag:[76,77],includ:[14,18,19,20,21],incred:80,index:75,indic:67,infer:[57,88],inherit:[3,4],inlin:76,input:47,instal:[81,85],int8:58,int8cachecalibr:3,int8calibr:4,ir:65,jetson:85,jit:67,languag:53,layer:65,learn:[51,52,53,54,56,57],lenet:56,level:[16,74,76,77],librari:[85,87],libtorchtrt:87,like:77,line:76,linear:60,link:[65,76],list:[42,43,44,45,77],liter:76,local:85,log:[18,22,23,24,25,26,27,28,39,42,69],logsoftmax:60,loop:60,lower:[60,62,64],macro:[19,43],make_int8_cache_calibr:30,make_int8_calibr:29,markup:76,mask:53,math:76,measur:57,menu:[78,80],meta:76,mlm:53,mod:75,model:[52,53,54,55,56,57,58,88],modul:[60,82,83],multibox:57,namespac:[18,19,20,21,38,39,40,41,49],nativ:85,native_op:65,nav:78,nest:[1,46],next:[51,52,53,54,55,56],node:59,number:[76,77],nvidia:67,object:[51,52,53,54,56,57],one:77,op:63,oper:[68,83],optim:88,optimz:60,option:[74,75,77],other:66,overview:[51,52,54,56,57,58,64],own:86,packag:[85,87],page:74,paragraph:[76,79],paramet:75,partit:[61,62,64],partitoninfo:61,pass:60,pattern:60,peephol:60,perform:58,phase:[59,60,61,62,63,64],plugin:87,post:86,pre:85,precis:[51,52,54],precompil:85,prerequisit:85,program:[42,43,44,45,87],project:74,ptq:[20,29,30,40,44,70,86],python:[65,67,82,84,85,86],pytorch:[56,65,67,90],quantiz:[58,86],queri:88,quickstart:83,quot:76,rabbit:77,read:65,redund:60,refer:[57,76],regist:83,relationship:[1,3,4,46],releas:85,remov:60,replac:76,resnet:54,respons:66,result:[57,63],right:85,rubric:76,runtim:[62,63,64,87],s:[51,52,53,54,55,56],sampl:[55,57],save:82,script:56,second:77,section:79,segmentedblock:61,serial:63,serv:88,server:88,set:[55,88],set_devic:35,set_is_colored_output_on:27,set_logging_prefix:28,set_reportable_log_level:25,setup:85,shape:55,shape_analysi:61,shot:57,sidebar:76,simpl:55,singl:[51,52,54,57],so:87,sometim:65,sourc:85,speedup:57,ssd:57,start:[51,52,54,56,67,83],step:88,sticki:78,str:5,struct:[46,47,48,49],structur:79,subdirectori:[13,14],submenu:78,submodul:71,subsect:79,subsubmenu:78,subsubsect:79,support:68,system:64,tabl:[74,75,76,77,78,79],tarbal:85,target:76,templat:[3,4,29,30],tensorformat:2,tensorrt:[49,51,52,54,55,56,57,58,63,65,67,83,84,85,87,88,90],test_py_modul:75,text:76,theme:[74,80],thi:[77,80],through:68,time:67,titl:76,tl:55,toc:74,topic:76,torch:[49,51,52,54,55,56,57,58,65,67,83,84,87,88,90],torch_tensorrt:[15,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,45,69,70,71,72],torch_tensorrt_major_vers:7,torch_tensorrt_minor_vers:8,torch_tensorrt_patch_vers:6,torch_tensorrt_vers:12,torchscript:[31,32,33,37,41,56,67,82,83],torchtrt_api:9,torchtrt_hidden:11,torchtrtc:[83,89],trace:56,train:[58,86],transform:53,triton:88,trt:55,ts:72,tupl:60,type:[3,4,46],under:83,unpack:60,unrol:60,unsupport:83,up:[55,88],us:[55,58,60,65,83,84,85,90],util:[51,55,57],version:63,via:81,visual:57,wai:76,weight:66,what:[51,52,53,54,55,56,66],wide:74,without:55,work:[55,82,83],write:66,xstr:10,your:[86,88]}}) \ No newline at end of file +Search.setIndex({docnames:["_cpp_api/classtorch__tensorrt_1_1DataType","_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType","_cpp_api/classtorch__tensorrt_1_1TensorFormat","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator","_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502","_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268","_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e","_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827","_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b","_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da","_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59","_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883","_cpp_api/dir_cpp","_cpp_api/dir_cpp_include","_cpp_api/dir_cpp_include_torch_tensorrt","_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558","_cpp_api/enum_torch__tensorrt_8h_1a3fbe5d72e4fc624dbd038853079620eb","_cpp_api/file_cpp_include_torch_tensorrt_logging.h","_cpp_api/file_cpp_include_torch_tensorrt_macros.h","_cpp_api/file_cpp_include_torch_tensorrt_ptq.h","_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h","_cpp_api/function_logging_8h_1a0593f776f469c20469e2f729fc7861a3","_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650","_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a","_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2","_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8","_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5","_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc","_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c","_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178","_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797","_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9","_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef","_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528","_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384","_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1","_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2","_cpp_api/namespace_torch_tensorrt","_cpp_api/namespace_torch_tensorrt__logging","_cpp_api/namespace_torch_tensorrt__ptq","_cpp_api/namespace_torch_tensorrt__torchscript","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h","_cpp_api/structtorch__tensorrt_1_1Device","_cpp_api/structtorch__tensorrt_1_1GraphInputs","_cpp_api/structtorch__tensorrt_1_1Input","_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec","_cpp_api/torch_tensort_cpp","_cpp_api/unabridged_orphan","cli/torchtrtc","contributors/conversion","contributors/lowering","contributors/partitioning","contributors/phases","contributors/runtime","contributors/system_overview","contributors/useful_links","contributors/writing_converters","getting_started/getting_started_with_cpp_api","getting_started/getting_started_with_python_api","getting_started/installation","index","indices/supported_ops","py_api/fx","py_api/logging","py_api/ptq","py_api/torch_tensorrt","py_api/ts","src/pytorch-sphinx-theme/docs/changelog","src/pytorch-sphinx-theme/docs/configuring","src/pytorch-sphinx-theme/docs/demo/api","src/pytorch-sphinx-theme/docs/demo/demo","src/pytorch-sphinx-theme/docs/demo/lists_tables","src/pytorch-sphinx-theme/docs/demo/long","src/pytorch-sphinx-theme/docs/demo/structure","src/pytorch-sphinx-theme/docs/index","src/pytorch-sphinx-theme/docs/installing","tutorials/creating_torchscript_module_in_python","tutorials/getting_started_with_fx_path","tutorials/notebooks","tutorials/ptq","tutorials/runtime","tutorials/serving_torch_tensorrt_with_triton","tutorials/use_from_pytorch","tutorials/using_dla"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1,nbsphinx:4,sphinx:56},filenames:["_cpp_api/classtorch__tensorrt_1_1DataType.rst","_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.rst","_cpp_api/classtorch__tensorrt_1_1TensorFormat.rst","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.rst","_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.rst","_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.rst","_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.rst","_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.rst","_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.rst","_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.rst","_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.rst","_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.rst","_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.rst","_cpp_api/dir_cpp.rst","_cpp_api/dir_cpp_include.rst","_cpp_api/dir_cpp_include_torch_tensorrt.rst","_cpp_api/enum_logging_8h_1a130f65408ad8cbaee060f05e8db69558.rst","_cpp_api/enum_torch__tensorrt_8h_1a3fbe5d72e4fc624dbd038853079620eb.rst","_cpp_api/file_cpp_include_torch_tensorrt_logging.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst","_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst","_cpp_api/function_logging_8h_1a0593f776f469c20469e2f729fc7861a3.rst","_cpp_api/function_logging_8h_1a0c012cb374addd90eb1f42eaec570650.rst","_cpp_api/function_logging_8h_1a56e110feaaba2c3fd44bd201fd21a76a.rst","_cpp_api/function_logging_8h_1a7cb50492421ea9de4e3db895819df6f2.rst","_cpp_api/function_logging_8h_1ac46ac0901cb97e3ae6e93b45f24e90b8.rst","_cpp_api/function_logging_8h_1ad2efd47b6c3689e58ccc595680579ae5.rst","_cpp_api/function_logging_8h_1af8f3443813315af7901903d25dd495cc.rst","_cpp_api/function_ptq_8h_1a226e3c83379d1012cde8578c1c86b16c.rst","_cpp_api/function_ptq_8h_1a6186e305f47c1d94b6130ef6c7f7e178.rst","_cpp_api/function_torch__tensorrt_8h_1a5b405fd3bf3c8fc2e2a54cbbab979797.rst","_cpp_api/function_torch__tensorrt_8h_1a6e19490a08fb1553c9dd347a5ae79db9.rst","_cpp_api/function_torch__tensorrt_8h_1a710df824a7718b440e4bc17bf9693cef.rst","_cpp_api/function_torch__tensorrt_8h_1ac4ab8313ae72c2c899ea31548b528528.rst","_cpp_api/function_torch__tensorrt_8h_1ad1acd06eaeaffbbcf6e7ebf426891384.rst","_cpp_api/function_torch__tensorrt_8h_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.rst","_cpp_api/function_torch__tensorrt_8h_1ae8d56472106eeef37fbe51ff7f40c9b2.rst","_cpp_api/namespace_torch_tensorrt.rst","_cpp_api/namespace_torch_tensorrt__logging.rst","_cpp_api/namespace_torch_tensorrt__ptq.rst","_cpp_api/namespace_torch_tensorrt__torchscript.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst","_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst","_cpp_api/structtorch__tensorrt_1_1Device.rst","_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst","_cpp_api/structtorch__tensorrt_1_1Input.rst","_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.rst","_cpp_api/torch_tensort_cpp.rst","_cpp_api/unabridged_orphan.rst","cli/torchtrtc.rst","contributors/conversion.rst","contributors/lowering.rst","contributors/partitioning.rst","contributors/phases.rst","contributors/runtime.rst","contributors/system_overview.rst","contributors/useful_links.rst","contributors/writing_converters.rst","getting_started/getting_started_with_cpp_api.rst","getting_started/getting_started_with_python_api.rst","getting_started/installation.rst","index.rst","indices/supported_ops.rst","py_api/fx.rst","py_api/logging.rst","py_api/ptq.rst","py_api/torch_tensorrt.rst","py_api/ts.rst","src/pytorch-sphinx-theme/docs/changelog.rst","src/pytorch-sphinx-theme/docs/configuring.rst","src/pytorch-sphinx-theme/docs/demo/api.rst","src/pytorch-sphinx-theme/docs/demo/demo.rst","src/pytorch-sphinx-theme/docs/demo/lists_tables.rst","src/pytorch-sphinx-theme/docs/demo/long.rst","src/pytorch-sphinx-theme/docs/demo/structure.rst","src/pytorch-sphinx-theme/docs/index.rst","src/pytorch-sphinx-theme/docs/installing.rst","tutorials/creating_torchscript_module_in_python.rst","tutorials/getting_started_with_fx_path.rst","tutorials/notebooks.rst","tutorials/ptq.rst","tutorials/runtime.rst","tutorials/serving_torch_tensorrt_with_triton.rst","tutorials/use_from_pytorch.rst","tutorials/using_dla.rst"],objects:{"":[[5,0,1,"c.STR","STR"],[9,0,1,"c.TORCHTRT_API","TORCHTRT_API"],[11,0,1,"c.TORCHTRT_HIDDEN","TORCHTRT_HIDDEN"],[7,0,1,"c.TORCH_TENSORRT_MAJOR_VERSION","TORCH_TENSORRT_MAJOR_VERSION"],[8,0,1,"c.TORCH_TENSORRT_MINOR_VERSION","TORCH_TENSORRT_MINOR_VERSION"],[6,0,1,"c.TORCH_TENSORRT_PATCH_VERSION","TORCH_TENSORRT_PATCH_VERSION"],[12,0,1,"c.TORCH_TENSORRT_VERSION","TORCH_TENSORRT_VERSION"],[10,0,1,"c.XSTR","XSTR"],[0,1,1,"_CPPv4N14torch_tensorrt8DataTypeE","torch_tensorrt::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value","torch_tensorrt::DataType::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE","torch_tensorrt::DataType::DataType"],[0,2,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEv","torch_tensorrt::DataType::DataType"],[0,3,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value","torch_tensorrt::DataType::DataType::t"],[0,3,1,"_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE","torch_tensorrt::DataType::DataType::t"],[0,4,1,"_CPPv4N14torch_tensorrt8DataType5ValueE","torch_tensorrt::DataType::Value"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kBoolE","torch_tensorrt::DataType::Value::kBool"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kCharE","torch_tensorrt::DataType::Value::kChar"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value6kFloatE","torch_tensorrt::DataType::Value::kFloat"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kHalfE","torch_tensorrt::DataType::Value::kHalf"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value4kIntE","torch_tensorrt::DataType::Value::kInt"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE","torch_tensorrt::DataType::Value::kUnknown"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kBoolE","torch_tensorrt::DataType::kBool"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kCharE","torch_tensorrt::DataType::kChar"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value6kFloatE","torch_tensorrt::DataType::kFloat"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value5kHalfE","torch_tensorrt::DataType::kHalf"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value4kIntE","torch_tensorrt::DataType::kInt"],[0,5,1,"_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE","torch_tensorrt::DataType::kUnknown"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypecv5ValueEv","torch_tensorrt::DataType::operator Value"],[0,2,1,"_CPPv4N14torch_tensorrt8DataTypecvbEv","torch_tensorrt::DataType::operator bool"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeneE8DataType","torch_tensorrt::DataType::operator!="],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE","torch_tensorrt::DataType::operator!="],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeneE8DataType","torch_tensorrt::DataType::operator!=::other"],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE","torch_tensorrt::DataType::operator!=::other"],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType","torch_tensorrt::DataType::operator=="],[0,2,1,"_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE","torch_tensorrt::DataType::operator=="],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType","torch_tensorrt::DataType::operator==::other"],[0,3,1,"_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE","torch_tensorrt::DataType::operator==::other"],[46,1,1,"_CPPv4N14torch_tensorrt6DeviceE","torch_tensorrt::Device"],[46,2,1,"_CPPv4N14torch_tensorrt6Device6DeviceEv","torch_tensorrt::Device::Device"],[1,1,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypeE","torch_tensorrt::Device::DeviceType"],[46,1,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypeE","torch_tensorrt::Device::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv","torch_tensorrt::Device::DeviceType::DeviceType"],[1,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType::t"],[1,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType::t"],[46,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value","torch_tensorrt::Device::DeviceType::DeviceType::t"],[46,3,1,"_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE","torch_tensorrt::Device::DeviceType::DeviceType::t"],[1,4,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE","torch_tensorrt::Device::DeviceType::Value"],[46,4,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE","torch_tensorrt::Device::DeviceType::Value"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::Value::kDLA"],[46,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::Value::kDLA"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::Value::kGPU"],[46,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::Value::kGPU"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE","torch_tensorrt::Device::DeviceType::kDLA"],[1,5,1,"_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE","torch_tensorrt::Device::DeviceType::kGPU"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv","torch_tensorrt::Device::DeviceType::operator Value"],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv","torch_tensorrt::Device::DeviceType::operator Value"],[1,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv","torch_tensorrt::Device::DeviceType::operator bool"],[46,2,1,"_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv","torch_tensorrt::Device::DeviceType::operator bool"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!="],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!="],[1,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!=::other"],[46,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType","torch_tensorrt::Device::DeviceType::operator!=::other"],[1,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator=="],[46,2,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator=="],[1,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator==::other"],[46,3,1,"_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType","torch_tensorrt::Device::DeviceType::operator==::other"],[46,6,1,"_CPPv4N14torch_tensorrt6Device18allow_gpu_fallbackE","torch_tensorrt::Device::allow_gpu_fallback"],[46,6,1,"_CPPv4N14torch_tensorrt6Device11device_typeE","torch_tensorrt::Device::device_type"],[46,6,1,"_CPPv4N14torch_tensorrt6Device8dla_coreE","torch_tensorrt::Device::dla_core"],[46,6,1,"_CPPv4N14torch_tensorrt6Device6gpu_idE","torch_tensorrt::Device::gpu_id"],[17,4,1,"_CPPv4N14torch_tensorrt16EngineCapabilityE","torch_tensorrt::EngineCapability"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE","torch_tensorrt::EngineCapability::kDLA_STANDALONE"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE","torch_tensorrt::EngineCapability::kSAFETY"],[17,5,1,"_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE","torch_tensorrt::EngineCapability::kSTANDARD"],[47,1,1,"_CPPv4N14torch_tensorrt11GraphInputsE","torch_tensorrt::GraphInputs"],[47,6,1,"_CPPv4N14torch_tensorrt11GraphInputs15input_signatureE","torch_tensorrt::GraphInputs::input_signature"],[47,6,1,"_CPPv4N14torch_tensorrt11GraphInputs6inputsE","torch_tensorrt::GraphInputs::inputs"],[48,1,1,"_CPPv4N14torch_tensorrt5InputE","torch_tensorrt::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input"],[48,2,1,"_CPPv4N14torch_tensorrt5Input5InputEv","torch_tensorrt::Input::Input"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::dtype"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::format"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::max_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::min_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::opt_shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat","torch_tensorrt::Input::Input::shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat","torch_tensorrt::Input::Input::shape"],[48,3,1,"_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE","torch_tensorrt::Input::Input::tensor"],[48,6,1,"_CPPv4N14torch_tensorrt5Input5dtypeE","torch_tensorrt::Input::dtype"],[48,6,1,"_CPPv4N14torch_tensorrt5Input6formatE","torch_tensorrt::Input::format"],[48,6,1,"_CPPv4N14torch_tensorrt5Input9max_shapeE","torch_tensorrt::Input::max_shape"],[48,6,1,"_CPPv4N14torch_tensorrt5Input9min_shapeE","torch_tensorrt::Input::min_shape"],[48,6,1,"_CPPv4N14torch_tensorrt5Input9opt_shapeE","torch_tensorrt::Input::opt_shape"],[48,6,1,"_CPPv4N14torch_tensorrt5Input5shapeE","torch_tensorrt::Input::shape"],[2,1,1,"_CPPv4N14torch_tensorrt12TensorFormatE","torch_tensorrt::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value","torch_tensorrt::TensorFormat::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE","torch_tensorrt::TensorFormat::TensorFormat"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEv","torch_tensorrt::TensorFormat::TensorFormat"],[2,3,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value","torch_tensorrt::TensorFormat::TensorFormat::t"],[2,3,1,"_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE","torch_tensorrt::TensorFormat::TensorFormat::t"],[2,4,1,"_CPPv4N14torch_tensorrt12TensorFormat5ValueE","torch_tensorrt::TensorFormat::Value"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE","torch_tensorrt::TensorFormat::Value::kChannelsLast"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE","torch_tensorrt::TensorFormat::Value::kContiguous"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE","torch_tensorrt::TensorFormat::Value::kUnknown"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE","torch_tensorrt::TensorFormat::kChannelsLast"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE","torch_tensorrt::TensorFormat::kContiguous"],[2,5,1,"_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE","torch_tensorrt::TensorFormat::kUnknown"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatcv5ValueEv","torch_tensorrt::TensorFormat::operator Value"],[2,2,1,"_CPPv4N14torch_tensorrt12TensorFormatcvbEv","torch_tensorrt::TensorFormat::operator bool"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat","torch_tensorrt::TensorFormat::operator!="],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator!="],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat","torch_tensorrt::TensorFormat::operator!=::other"],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator!=::other"],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat","torch_tensorrt::TensorFormat::operator=="],[2,2,1,"_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator=="],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat","torch_tensorrt::TensorFormat::operator==::other"],[2,3,1,"_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE","torch_tensorrt::TensorFormat::operator==::other"],[36,2,1,"_CPPv4N14torch_tensorrt15dump_build_infoEv","torch_tensorrt::dump_build_info"],[34,2,1,"_CPPv4N14torch_tensorrt14get_build_infoEv","torch_tensorrt::get_build_info"],[16,4,1,"_CPPv4N14torch_tensorrt7logging5LevelE","torch_tensorrt::logging::Level"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE","torch_tensorrt::logging::Level::kDEBUG"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kERRORE","torch_tensorrt::logging::Level::kERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE","torch_tensorrt::logging::Level::kGRAPH"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level5kINFOE","torch_tensorrt::logging::Level::kINFO"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE","torch_tensorrt::logging::Level::kINTERNAL_ERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE","torch_tensorrt::logging::Level::kWARNING"],[24,2,1,"_CPPv4N14torch_tensorrt7logging24get_is_colored_output_onEv","torch_tensorrt::logging::get_is_colored_output_on"],[22,2,1,"_CPPv4N14torch_tensorrt7logging18get_logging_prefixEv","torch_tensorrt::logging::get_logging_prefix"],[23,2,1,"_CPPv4N14torch_tensorrt7logging24get_reportable_log_levelEv","torch_tensorrt::logging::get_reportable_log_level"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE","torch_tensorrt::logging::kDEBUG"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kERRORE","torch_tensorrt::logging::kERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE","torch_tensorrt::logging::kGRAPH"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level5kINFOE","torch_tensorrt::logging::kINFO"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE","torch_tensorrt::logging::kINTERNAL_ERROR"],[16,5,1,"_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE","torch_tensorrt::logging::kWARNING"],[26,2,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log"],[26,3,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log::lvl"],[26,3,1,"_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE","torch_tensorrt::logging::log::msg"],[27,2,1,"_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb","torch_tensorrt::logging::set_is_colored_output_on"],[27,3,1,"_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb","torch_tensorrt::logging::set_is_colored_output_on::colored_output_on"],[28,2,1,"_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE","torch_tensorrt::logging::set_logging_prefix"],[28,3,1,"_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE","torch_tensorrt::logging::set_logging_prefix::prefix"],[25,2,1,"_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level","torch_tensorrt::logging::set_reportable_log_level"],[25,3,1,"_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level","torch_tensorrt::logging::set_reportable_log_level::lvl"],[3,1,1,"_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE","torch_tensorrt::ptq::Int8CacheCalibrator"],[3,7,1,"_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE","torch_tensorrt::ptq::Int8CacheCalibrator::Algorithm"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE","torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE","torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator::cache_file_path"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::bindings"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::names"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8CacheCalibrator::getBatch::nbBindings"],[3,2,1,"_CPPv4NK14torch_tensorrt3ptq19Int8CacheCalibrator12getBatchSizeEv","torch_tensorrt::ptq::Int8CacheCalibrator::getBatchSize"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibratorcvPN8nvinfer115IInt8CalibratorEEv","torch_tensorrt::ptq::Int8CacheCalibrator::operator nvinfer1::IInt8Calibrator*"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::readCalibrationCache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::readCalibrationCache::length"],[3,2,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache::cache"],[3,3,1,"_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8CacheCalibrator::writeCalibrationCache::length"],[4,1,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator"],[4,7,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator::Algorithm"],[4,7,1,"_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE","torch_tensorrt::ptq::Int8Calibrator::DataLoaderUniquePtr"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::cache_file_path"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::dataloader"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb","torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::use_cache"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::bindings"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::names"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator8getBatchEA_PvA_PKci","torch_tensorrt::ptq::Int8Calibrator::getBatch::nbBindings"],[4,2,1,"_CPPv4NK14torch_tensorrt3ptq14Int8Calibrator12getBatchSizeEv","torch_tensorrt::ptq::Int8Calibrator::getBatchSize"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8CalibratorcvPN8nvinfer115IInt8CalibratorEEv","torch_tensorrt::ptq::Int8Calibrator::operator nvinfer1::IInt8Calibrator*"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8Calibrator::readCalibrationCache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator20readCalibrationCacheER6size_t","torch_tensorrt::ptq::Int8Calibrator::readCalibrationCache::length"],[4,2,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache::cache"],[4,3,1,"_CPPv4N14torch_tensorrt3ptq14Int8Calibrator21writeCalibrationCacheEPKv6size_t","torch_tensorrt::ptq::Int8Calibrator::writeCalibrationCache::length"],[29,2,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator"],[29,7,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator::Algorithm"],[29,3,1,"_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE","torch_tensorrt::ptq::make_int8_cache_calibrator::cache_file_path"],[30,2,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator"],[30,7,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::Algorithm"],[30,7,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::DataLoader"],[30,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::cache_file_path"],[30,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::dataloader"],[30,3,1,"_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb","torch_tensorrt::ptq::make_int8_calibrator::use_cache"],[35,2,1,"_CPPv4N14torch_tensorrt10set_deviceEKi","torch_tensorrt::set_device"],[35,3,1,"_CPPv4N14torch_tensorrt10set_deviceEKi","torch_tensorrt::set_device::gpu_id"],[49,1,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpecE","torch_tensorrt::torchscript::CompileSpec"],[49,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[49,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[49,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[49,2,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec"],[49,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"],[49,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"],[49,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::input_signature"],[49,3,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE","torch_tensorrt::torchscript::CompileSpec::CompileSpec::inputs"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec10capabilityE","torch_tensorrt::torchscript::CompileSpec::capability"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec5debugE","torch_tensorrt::torchscript::CompileSpec::debug"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec6deviceE","torch_tensorrt::torchscript::CompileSpec::device"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec12disable_tf32E","torch_tensorrt::torchscript::CompileSpec::disable_tf32"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec20dla_global_dram_sizeE","torch_tensorrt::torchscript::CompileSpec::dla_global_dram_size"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec19dla_local_dram_sizeE","torch_tensorrt::torchscript::CompileSpec::dla_local_dram_size"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec13dla_sram_sizeE","torch_tensorrt::torchscript::CompileSpec::dla_sram_size"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec18enabled_precisionsE","torch_tensorrt::torchscript::CompileSpec::enabled_precisions"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec12graph_inputsE","torch_tensorrt::torchscript::CompileSpec::graph_inputs"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14min_block_sizeE","torch_tensorrt::torchscript::CompileSpec::min_block_size"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec20num_avg_timing_itersE","torch_tensorrt::torchscript::CompileSpec::num_avg_timing_iters"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14ptq_calibratorE","torch_tensorrt::torchscript::CompileSpec::ptq_calibrator"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec5refitE","torch_tensorrt::torchscript::CompileSpec::refit"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec24require_full_compilationE","torch_tensorrt::torchscript::CompileSpec::require_full_compilation"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14sparse_weightsE","torch_tensorrt::torchscript::CompileSpec::sparse_weights"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec22torch_executed_modulesE","torch_tensorrt::torchscript::CompileSpec::torch_executed_modules"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec18torch_executed_opsE","torch_tensorrt::torchscript::CompileSpec::torch_executed_ops"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec24truncate_long_and_doubleE","torch_tensorrt::torchscript::CompileSpec::truncate_long_and_double"],[49,6,1,"_CPPv4N14torch_tensorrt11torchscript11CompileSpec14workspace_sizeE","torch_tensorrt::torchscript::CompileSpec::workspace_size"],[31,2,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support"],[31,3,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support::method_name"],[31,3,1,"_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE","torch_tensorrt::torchscript::check_method_operator_support::module"],[32,2,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile"],[32,3,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile::info"],[32,3,1,"_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec","torch_tensorrt::torchscript::compile::module"],[37,2,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::info"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::method_name"],[37,3,1,"_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec","torch_tensorrt::torchscript::convert_method_to_trt_engine::module"],[33,2,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module"],[33,3,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module::device"],[33,3,1,"_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6Device","torch_tensorrt::torchscript::embed_engine_in_new_module::engine"],[69,8,0,"-","torch_tensorrt"]],"torch_tensorrt.Device":[[69,10,1,"","__init__"],[69,11,1,"","allow_gpu_fallback"],[69,11,1,"","device_type"],[69,11,1,"","dla_core"],[69,11,1,"","gpu_id"]],"torch_tensorrt.Input":[[69,10,1,"","__init__"],[69,11,1,"","dtype"],[69,11,1,"","format"],[69,11,1,"","shape"],[69,11,1,"","shape_mode"]],"torch_tensorrt.fx":[[66,9,1,"","InputTensorSpec"],[66,9,1,"","TRTInterpreter"],[66,9,1,"","TRTInterpreterResult"],[66,9,1,"","TRTModule"]],"torch_tensorrt.logging":[[67,9,1,"","Level"],[67,9,1,"","debug"],[67,9,1,"","errors"],[67,12,1,"","get_is_colored_output_on"],[67,12,1,"","get_logging_prefix"],[67,12,1,"","get_reportable_log_level"],[67,9,1,"","graphs"],[67,9,1,"","info"],[67,9,1,"","internal_errors"],[67,12,1,"","log"],[67,12,1,"","set_is_colored_output_on"],[67,12,1,"","set_logging_prefix"],[67,12,1,"","set_reportable_log_level"],[67,9,1,"","warnings"]],"torch_tensorrt.logging.Level":[[67,11,1,"","Debug"],[67,11,1,"","Error"],[67,11,1,"","Graph"],[67,11,1,"","Info"],[67,11,1,"","InternalError"],[67,11,1,"","Warning"]],"torch_tensorrt.ptq":[[68,9,1,"id1","CacheCalibrator"],[68,9,1,"id2","CalibrationAlgo"],[68,9,1,"id0","DataLoaderCalibrator"],[68,12,1,"","get_batch"],[68,12,1,"","get_batch_size"],[68,12,1,"","get_cache_mode_batch"],[68,12,1,"","read_calibration_cache"],[68,12,1,"","write_calibration_cache"]],"torch_tensorrt.ptq.CacheCalibrator":[[68,10,1,"","__init__"]],"torch_tensorrt.ptq.CalibrationAlgo":[[68,11,1,"","ENTROPY_CALIBRATION"],[68,11,1,"","ENTROPY_CALIBRATION_2"],[68,11,1,"","LEGACY_CALIBRATION"],[68,11,1,"","MINMAX_CALIBRATION"]],"torch_tensorrt.ptq.DataLoaderCalibrator":[[68,10,1,"","__init__"]],"torch_tensorrt.ts":[[70,12,1,"","TensorRTCompileSpec"],[70,12,1,"","check_method_op_support"],[70,12,1,"","compile"],[70,12,1,"","convert_method_to_trt_engine"],[70,12,1,"","embed_engine_in_new_module"]],torch_tensorrt:[[69,9,1,"","Device"],[69,9,1,"","DeviceType"],[69,9,1,"","EngineCapability"],[69,9,1,"","Input"],[69,9,1,"","TensorFormat"],[69,12,1,"","compile"],[69,12,1,"","convert_method_to_trt_engine"],[69,9,1,"","dtype"],[69,12,1,"","dump_build_info"],[66,8,0,"-","fx"],[69,12,1,"","get_build_info"],[67,8,0,"-","logging"],[68,8,0,"-","ptq"],[69,12,1,"","set_device"],[70,8,0,"-","ts"]]},objnames:{"0":["c","macro","C macro"],"1":["cpp","class","C++ class"],"10":["py","method","Python method"],"11":["py","attribute","Python attribute"],"12":["py","function","Python function"],"2":["cpp","function","C++ function"],"3":["cpp","functionParam","C++ function parameter"],"4":["cpp","enum","C++ enum"],"5":["cpp","enumerator","C++ enumerator"],"6":["cpp","member","C++ member"],"7":["cpp","templateParam","C++ template parameter"],"8":["py","module","Python module"],"9":["py","class","Python class"]},objtypes:{"0":"c:macro","1":"cpp:class","10":"py:method","11":"py:attribute","12":"py:function","2":"cpp:function","3":"cpp:functionParam","4":"cpp:enum","5":"cpp:enumerator","6":"cpp:member","7":"cpp:templateParam","8":"py:module","9":"py:class"},terms:{"0":[33,43,44,45,49,52,58,60,61,63,65,67,68,69,70,71,73,74,81,83,85,86,87],"0000":75,"01":[61,65,75],"0208":61,"03":75,"0358":61,"0383":61,"04":[61,85],"0435":61,"0464":61,"0530":61,"0678":61,"0805":61,"0818":61,"0932":61,"0a0":74,"0baf50a":74,"0x7f3b810fa3f0":70,"1":[3,4,33,43,44,45,48,49,52,54,55,57,60,61,62,63,65,66,67,68,69,70,71,72,74,75,78,80,81,82,83,86,87],"10":[49,61,63,70,78,80,82,83,85],"100":[66,81],"1000":85,"1012":54,"1013":54,"1024":[52,69,70,82],"1045":61,"1048576":[45,49,70],"1056":61,"1063":61,"1073741824":[45,49,70],"109":61,"11":[54,61,63,74,78,85],"119":80,"12":[54,61,74,78,80,85],"120":[61,80],"123":75,"129":80,"13":[74,78],"136":85,"137":80,"138":80,"14":[78,85],"1409":83,"15":[74,78],"1502":61,"1549":61,"1556":83,"16":[61,62,69,78,80],"1691":61,"17":78,"18":[61,78],"19":[75,78],"1994":83,"1d":54,"1e":52,"2":[33,43,55,60,61,63,65,67,68,69,70,72,74,75,78,80,81,83],"20":78,"2009":83,"2010":83,"2012":75,"2014":83,"2020":[61,64],"2023":83,"22":85,"224":[55,66,69,70,82,85],"225":[66,85],"229":85,"23":[49,54,70,75],"234375":85,"24":54,"244":[69,70],"248":54,"249":54,"25":[61,66,81],"256":85,"258":74,"27":61,"28":61,"2802":61,"2822":74,"287":74,"29":61,"2c3":75,"3":[45,49,52,54,55,57,61,63,65,67,68,69,70,74,75,78,80,81,82,83,86,87],"300":[52,86],"31":61,"32":[52,61,62,69,80,83,87],"320":83,"32bit":52,"33":61,"346":61,"35":61,"36":61,"3677":54,"37":61,"38":80,"39":80,"3d":81,"4":[57,61,63,65,67,72,74,75,78,81],"406":85,"429688":85,"4465":83,"456":85,"468750":85,"4822":83,"485":85,"4914":83,"5":[52,57,58,61,63,67,69,74,75,78,80,81,85],"50":82,"512":[52,69,70,82],"523438":85,"53":75,"536870912":[45,49,70],"539":61,"56":61,"576":61,"6":[54,57,61,63,65,78,80],"622":54,"64":[62,81],"64bit":52,"664062":85,"7":[57,58,61,78],"72048":63,"7302":75,"8":[3,52,54,61,63,69,74,75,78,85],"8000":85,"8001":85,"8002":85,"84":[61,80],"9":[61,78,85],"90":85,"92":85,"9223372036854775807":65,"abstract":[57,60,75],"boolean":[69,81],"break":[74,81],"byte":[69,70,82],"case":[0,1,2,46,49,53,57,60,63,81,83,84],"catch":[54,61],"char":[3,4,44,52,61],"class":[17,29,30,44,45,46,51,57,60,61,62,67,74,75,80,81,82,83],"const":[0,1,2,3,4,29,30,31,32,33,35,37,44,45,46,54,60,61,65,83],"default":[0,1,2,3,4,16,29,30,43,45,46,48,49,52,55,61,62,63,66,69,70,72,73,74,81,83,86],"do":[53,54,55,60,61,62,73,75,80,81,83,87],"enum":[0,1,2,42,45,46,51,67,70,83],"export":63,"final":[53,56,58,63,82],"float":[49,52,61,62,65,69,80,83,86],"function":[0,1,2,3,4,46,48,49,51,54,55,57,60,61,63,80,81,82,83,85,86,87],"import":[52,54,55,61,62,63,72,74,80,81,84,85,86],"int":[0,3,4,35,44,45,49,52,61,65,66,69,70,72],"long":[49,52,53,74,75],"new":[0,1,2,3,4,32,33,46,48,49,57,58,60,61,67,70,74,81,85],"public":[0,1,2,3,4,44,45,46,47,48,49,75,83],"return":[0,1,2,3,4,23,24,29,30,31,32,33,34,37,42,43,44,45,46,54,56,57,58,60,61,62,67,69,70,80,81,83,85],"short":[54,74,75],"static":[48,49,53,60,61,69,70,72],"super":[44,80],"throw":[52,54,61],"true":[0,1,2,4,46,49,54,55,60,61,65,66,69,70,72,75,81,83,85,86,87],"try":[58,61,74,75,86],"var":65,"void":[3,4,25,26,27,28,35,36,42,44,45],"while":[63,82,83,85],A:[4,29,30,32,33,47,48,54,55,60,63,70,75,83,85],And:61,As:[61,81],At:73,But:[61,74],By:[29,30,51,55,72,80],For:[53,55,61,63,66,72,74,75,80,81,82,83,84,85,86],If:[27,53,54,61,62,63,66,67,69,72,74,81,83,84,85,87],In:[0,1,2,46,53,56,57,58,60,62,63,64,74,75,77,81,82,83,84,85],Is:[24,69],It:[52,54,55,56,58,60,63,72,74,81,82],Its:[60,74],Not:3,On:55,One:[61,74,75,81,82],Or:74,THE:74,TO:61,That:74,Thats:61,The:[1,46,48,49,52,53,54,55,56,57,58,60,62,63,67,69,70,72,75,80,81,82,83,85,86],Then:[55,63,83,86],There:[4,53,58,60,63,75,80,81,82,83,84,85],These:[53,57,72,74,83,85],To:[1,46,52,55,61,62,63,72,80,85,86],Will:31,With:[61,72,74,83,85],_:[74,81],___torch_mangle_10:80,___torch_mangle_4847:57,___torch_mangle_5:80,___torch_mangle_9:80,__and__:65,__attribute__:43,__getitem__:65,__gnuc__:43,__init__:[68,69,74,80],__is__:65,__isnot__:65,__not__:65,__or__:65,__range_length:65,__round_to_zero_floordiv:65,__torch__:[57,61,80],__torch___pytorch_detection_ssd_src_model_ssd300_trt_engin:57,__torch___torchvision_models_resnet____torch_mangle_4847_resnet_trt_engin:57,__visibility__:43,__xor__:65,_all_:54,_c:[70,86],_convolut:[61,65],_jit_to_backend:86,_script:70,_shapemod:69,_theme:79,_validate_not_a_forked_repo:85,a1b:75,aarch64:58,ab:65,abi:84,abl:[53,54,60,64,81,83,86],about:[52,53,57,60,61,63,69,72,85],abov:[25,61,63,67,73,74,81],absolut:52,ac:77,acc_mod:81,acc_norm:81,acc_op:81,acc_op_convert:81,acc_ops_sigmoid:81,acc_trac:81,acceler:87,accept:[48,52,57,60,61,62,69],access:[54,60,61,64,72,81,86],accord:[60,70],accordingli:[72,81],account:85,accumsan:77,accumul:[49,70],accuraci:[82,83],achiev:82,aco:65,acosh:65,acoust:82,acquir:61,across:[49,52,54,70,72],acthardtanh:60,action:[74,81],activ:[61,70,74,81,82,83,87],activationtyp:[60,81],actual:[54,57,60,61,67,80,81],ad:[25,52,53,81],adaptive_avg_pool1d:65,adaptive_avg_pool2d:65,adaptive_avg_pool3d:65,adaptive_max_pool1d:65,adaptive_max_pool2d:65,adaptive_max_pool3d:65,add:[26,53,54,55,60,61,62,63,65,67,72,74,79],add_:[54,61,65],add_activ:81,addactiv:60,addit:[54,61,69,81,82],addlay:61,address:75,addshuffl:61,adipisc:[75,77],adjac:74,adjust:74,adopt:82,advanc:[75,83],advis:74,aenean:77,afford:81,aforement:85,after:[52,53,54,55,61,62,64,80,81,84,85],again:[44,57,60,74],against:[52,61],agx:45,ahead:61,aim:54,algo_typ:[68,83],algorithm:[3,4,29,30,44,68,81,83],algorithm_selector:81,alias:43,align:74,align_corn:65,aliquam:77,aliquet:[75,77],all:[16,42,43,44,45,49,52,54,55,57,61,62,63,67,69,74,75,80,81,82,83,84,85],alloc:60,allow:[48,49,52,53,54,69,72,81],allow_gpu_fallback:[45,46,69,70,83,86,87],allow_tf32:65,almost:61,alpha:[65,75,81],alreadi:[52,53,54,61,83],also:[29,53,60,61,62,63,64,72,74,75,82,83],altern:[48,82],although:74,altogeth:[55,72],alwai:[3,4,27,52,74],amet:[75,77],an:[2,3,4,48,49,52,53,54,55,56,57,58,60,61,62,63,64,68,69,70,72,74,75,80,81,82,83,84,85],analogu:60,analysi:55,analyt:72,analytics_id:72,ancient:74,ani:[48,52,53,60,61,62,63,69,72,74,81,83],ann:74,annot:[60,61],anonym:74,anoth:[62,74,75,80],ant:77,anyon:75,anyth:[74,75,84],aot:[61,64],api:[55,58,60,61,62,69,70,73,81,82,83,84,85,86],appear:74,append:65,appli:83,applic:[1,29,46,52,54,58,61,62,84,86,87],apr:61,ar:[42,46,49,52,53,54,55,57,58,60,61,63,64,69,70,72,74,75,76,80,81,82,83,84,85,86],arab:75,arang:65,architectur:[63,64,82],archiv:63,arcu:[75,77],area:76,aren:61,arg:[53,61,68,69,78,81,82],arg_replacement_tupl:81,argc:61,argument:[48,52,54,57,60,61,69,70,74,75,81],argv:61,around:[54,57,60,74,77,80],arrai:[3,4,33,53,70],arrayref:[45,48,49],arxiv:83,as_numpi:85,asin:65,asinh:65,aspect:52,assembl:[53,61],assign:[3,4,73],associ:[53,60,61],associatevalueandivalu:60,associatevalueandtensor:[60,61],assum:86,atan:65,atanh:65,aten:[49,54,55,59,60,61,65,70],atol:52,attrdict:85,attribut:[54,55,57,61,74,81],auctor:77,audio:82,augu:77,author:75,auto:[44,55,60,61,74,75,83,87],autodoc:[74,75],automat:[61,74],avail:[52,60,63,72,81,87],averag:[49,52,70],avg:52,avg_pool1d:65,avg_pool2d:65,avg_pool3d:65,awai:74,awaken:74,axi:65,b0:82,b:[63,65,75,85],b_hh:65,b_ih:65,back:[54,55,57,58,61,69,74,80],back_insert:44,backend:[70,73,86],background:[74,80],backlink:74,backward:81,bar:[72,74],base:[36,50,57,63,67,68,69,74,80,82,83],bash:63,basi:74,basic:[52,75,81,85],batch:[3,4,44,66,81,83,85,87],batch_norm:[60,65],batch_siz:[44,83],batched_data_:44,batchnorm:54,batchtyp:44,bathroom:74,bazel:[58,63],bazel_vers:63,bazelbuild:63,bazelisk:63,bazelvers:63,bdist_wheel:63,beat:75,becaus:[60,61,63,66,80,81],becom:60,bee:74,been:[53,60,61,75],befor:[49,54,58,60,61,63,64,70,81,85],beforehand:61,begin:[44,63,74,81],beginn:80,begun:74,behav:76,behavior:[49,69,70,81],behind:74,being:[61,81],belong:74,below:[60,61,63,74,81,85],benchmark:65,benefit:[60,61],besid:74,best:[63,74,81],beta:[65,70,81],better:[80,82],between:[54,60,63,74,75,83],bia:[54,61,65],bibendum:77,bibliograph:75,bigger:74,bin:63,binari:[44,83],binary_data:85,bind:[3,4,33,44,70,74],bird:85,bit:[49,60,61,69,70,81],bitbucket:72,bitbucket_url:72,blandit:77,blank:74,blob:[59,72,83],block0:54,block1:54,block:[52,53,54,78],blue:74,bmm:65,bodi:[74,75],bold:74,bool:[0,1,2,3,4,24,27,30,31,42,44,45,46,49,54,60,61,65,66,67,69,70,72,83],border:74,both:[63,72,74,80,83],bottom:72,box:74,bracket:74,branch:63,bread:74,brief:55,briefli:80,brontosaurus:74,browser:74,bsd:[42,43,44,45],buffer:[3,4,81],bug:63,bui:75,build:[29,30,34,49,52,53,56,58,60,61,69,73,78,81,83],build_fil:63,build_model:81,builder:81,builderconfig:45,built:[33,52,57,58,63,70],builtin:81,button:[72,74],bytearrai:81,c10:[0,1,45,46,48,49,61,83],c:[42,43,44,45,52,58,62,65,75,84,85,87],c_api:59,c_str:[60,61],cach:[3,4,29,30,44,52,61,68,81,83],cache_:44,cache_fil:[44,68,83],cache_file_path:[3,4,29,30,44],cache_file_path_:44,cache_size_:44,cachecalibr:[68,83],cackl:75,calcul:[48,53,55,61],calibr:[3,4,29,30,44,49,52,61,68,70,83],calibration_cache_fil:[29,30,83],calibration_dataload:[30,83],calibration_dataset:83,calibrationalgo:[68,83],call:[29,30,32,49,54,57,60,61,70,74,80,81,82,86],call_funct:81,callmethod:80,can:[0,1,4,29,30,37,46,47,48,49,52,53,54,55,56,57,58,60,61,62,63,69,70,72,74,80,81,82,83,84,85,86],canada:75,cannot:[48,54,55,63,69,70,73,80,81],canon:72,canonical_url:72,capabl:[17,45,49,52,57,69,70,86],capit:74,caption:[74,77],cast:[3,4,54],cat:[63,65],caught:54,caus:[60,63,72],cd:[63,85],cdll:61,ceil:65,ceil_mod:65,cell:75,centercrop:85,cerr:61,certain:[63,81],cfg:55,chain:60,challeng:85,chanc:60,chang:[29,54,58,70,72,81,83,85],changelog:78,channel:[2,69,73],channel_last:[69,70,82],channels_last:69,charact:74,check:[0,1,31,46,52,54,60,61,63,70,81,84,85],check_method_op_support:70,check_method_operator_support:[21,41,45,50],checkmethodoperatorsupport:61,child:75,children:81,choic:[63,68],choos:[80,81],cifar10:83,cifar:83,clamp:65,clamp_max:65,clamp_min:65,class_count:85,classif:[61,80,82],classifi:[75,82],classification_index:85,clean:74,clear:44,cli:[52,62],clickabl:74,clone:65,close:61,closer:54,closet:74,cnn:82,co:[65,75,82],code:[52,55,58,61,64,73,75,80,81,83],coeffici:82,collapse_navig:72,collat:75,collect:[61,70],colon:74,color:[24,27,67,74],colored_output_on:[27,42,67],column:75,com:[59,61,63,83,84,85],combin:81,come:[63,73,81,85],command:[52,61,63,74,75,80,85],comment:[63,74],commodo:77,common:[53,54,66,74,81],common_subexpression_elimin:54,commonli:75,commun:[49,52,61,70],compar:[62,81],comparis:[0,2],comparison:[1,46],compat:[0,1,46,54,57,63,70,81],compil:[21,31,37,41,45,49,50,52,54,55,57,60,62,67,69,70,72,80,81,83,84,85,86,87],compile_spec:[83,87],compilegraph:[61,83],compilesepc:33,compilespec:[3,4,21,32,37,41,45,50,55,61,83,87],compilespecstruct:50,complet:[55,61,80],complex:[47,49,63,80],complianc:52,compliat:83,complic:63,compon:[56,58,80,84],compos:[80,81,83,85],composit:61,compound:82,comput:[49,74,81,82,83],conceiv:74,concorr:85,condimentum:77,condit:74,conf:[72,79],confidence_scor:85,config:[63,85],configur:[32,37,48,61,63,64,69,70,78,83,85],congu:77,connect:[54,70,74,85,87],consectetur:[75,77],consecut:55,consid:[61,70],consider:85,consist:[54,74,81],consol:52,consolid:80,constant:[53,54,61],constant_pad_nd:65,constexpr:[0,1,2,45,46],construct:[0,1,2,3,4,46,48,49,53,54,56,58,60,61,68,69,74,75,81,83],constructor:[0,2,46,48,49,57,80],consult:73,consum:[4,53,80],contact:75,contain:[30,31,52,53,54,60,61,63,66,69,74,75,80,81,83,84,85],content:[78,83,85],context:[53,56,57,58,67],contextnet:82,contigu:[2,48,49,52,69,70],continu:[74,81,84],contributor:61,control:[80,81],conv1:[61,80],conv2:[61,80],conv2d:80,conv:[49,52,61],conval:77,convect:48,conveni:[82,83],converison:81,convers:[54,55,57,61,69,70,81],conversionctx:[60,61],convert:[3,4,31,32,37,52,54,55,56,58,62,64,69,70,82,84,86],convert_method_to_trt_engin:[21,41,45,50,69,70,86],convertgraphtotrtengin:61,convien:49,convienc:[3,4,49],convolut:[70,83,87],convtert:81,coordin:58,copi:[44,60,65,68,75,81,85],copy_:65,copyright:[42,43,44,45,61,75],core:[45,52,54,55,58,61,69,87],core_id:69,corpor:[42,43,44,45],corpu:82,correct:[57,63,72],correctli:63,correspond:[60,63,81],cosh:65,could:81,count_include_pad:65,coupl:[53,58,81,84],cout:61,cover:82,cp:63,cpp:[14,15,42,43,44,45,51,54,58,61,63,83],cpp_frontend:83,cppdirectori:50,cppdoc:61,cpu:66,cra:77,creat:[29,30,33,52,53,57,60,61,64,70,74,81,85],credit:61,criteria:[55,56,58],cross:74,cs:83,csrc:[54,59],cstddef:83,ctx:[60,61],ctype:61,cuda113:63,cuda:[49,57,61,62,63,66,69,83,85,86],cuda_graph_batch_s:66,cuda_runtim:[21,45],cudafloattyp:61,cudasetdevic:35,cudnn_en:65,cumsum:65,curabitur:77,curl:[63,74],current:[23,57,60,63,70,72,81],cursu:77,custom:[52,63,81],custom_class:[21,45],custom_mapp:81,customclasshold:[45,48],cut:74,cxx11:84,d:[52,74,75,87],dapibu:77,data:[0,2,3,4,29,30,44,46,48,49,52,53,55,56,58,60,65,68,69,70,74,78,82,83],data_dir:83,data_item_1:73,data_typ:85,dataclass:81,dataflow:[60,61],dataload:[4,29,30,44,49,68,83],dataloader_:44,dataloadercalibr:[68,83],dataloaderopt:83,dataloaderuniqueptr:[4,44],dataset:[29,68,82,83],datatyp:[1,21,38,45,46,48,49,50,62,69,70,85],datatypeclass:50,date:75,david:75,dbg:63,dcmake_build_typ:63,dcmake_module_path:63,dead_code_elimin:54,deal:60,debug:[16,27,45,49,52,60,67,70,86],debugg:[52,70],decid:69,declar:63,deconvolut:87,decor:81,dedic:[54,75],deep:[60,64,72,83,87],deeplearn:[59,81],def:[74,80,81,85],defin:[0,1,2,3,4,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,43,46,47,48,49,51,52,61,62,69,72,80,81,82,83],definit:[51,60,74],deiti:74,delet:[0,1,2,45,46,54],delimit:54,demo:[74,83],demonstr:[74,75,76,82,83,85],demostr:82,denot:74,dep:63,depend:[29,34,53,55,58,61,62,81,84,85],depickl:57,deploi:[56,58,61,64,83,85],deploy:[52,61,62,82,83,84,85,87],deprec:[65,81],depth:[72,82],descclassnam:74,descnam:74,describ:[49,60,80,85,86],descript:[55,75],deseri:[61,69,70],design:[81,82,87],desir:[75,83],destini:75,destroi:[60,75],destructor:60,detail:[61,80,81,84,85],detect:[48,57],determin:[54,81],determinist:65,develop:[61,63,64,74,75,81],deviat:52,devic:[21,33,35,38,45,49,50,52,57,62,65,66,68,69,70,82,83,86,87],device_typ:[45,46,69,83,86,87],deviceclass:50,devicetyp:[21,38,45,46,50,69,83,86,87],devicetypestruct:50,diam:77,dict:[69,70],dictionari:[69,70,86],dictum:77,dictumst:77,did:74,didn:74,differ:[29,54,58,63,64,72,80,81],dignissim:77,dilat:65,dim0:65,dim1:65,dim:[65,85],dim_int:65,dim_intlist:65,dimens:[48,54,66,81,82],direct:[78,84],directli:[60,63,64,68,83],directori:[18,19,20,21,42,43,44,45,50,63,83],disabl:[52,67,72,73],disable_tf32:[45,49,70,83],disclos:63,disconnect:74,discret:74,discuss:85,displai:[52,67,72],display_github:72,display_gitlab:72,display_vers:72,dist:63,distdir:63,distribut:[61,69,83,84],div:65,div_:65,divisor_overrid:65,django:73,dl:74,dl_open:84,dla:[1,45,46,49,52,64,69,70],dla_cor:[45,46,52,69,83,86,87],dla_global_dram_s:[45,49,52,70],dla_local_dram_s:[45,49,52,70],dla_sram_s:[45,49,52,70],dla_standalon:52,dlacor:52,dll:52,doc:[58,59,63,72,73,74,79],docker:85,docsrc:58,docstr:[74,75],document:[42,43,44,45,50,58,61,72,74,75,79,80,83,84,85,86],docutil:[74,75],doe:[43,44,54,55,60,74,81,83],doesn:[61,63,74,80],dolor:[75,77],domain:[75,83],don:[60,72,74,75,81,83,85],done:[53,55,58,85],donec:[75,77],dont:42,dothismethod:74,dotpai:73,dotpayprovid:73,doubl:[49,52,70,74],down:[63,72,81],download:[63,78,83,85],downstream:82,doxygen_should_skip_thi:[44,45],dpython:[69,70],dram:52,dream:75,driver:63,drop:[63,72],dt:74,dtensorrt_root:63,dtorch_dir:63,dtyep:66,dtype:[45,48,49,52,62,65,66,69,70,81,82],dual:74,due:[3,4,63,73,74],dui:[75,77],dump:[36,52,63],dump_build_info:[21,38,45,50,69],durat:74,dure:[49,52,60,68,82,83,84],dynam:[48,49,66,69,70,81],dynamic_batch:81,e:[29,30,52,54,60,61,63,66,69,80,81,83],each:[3,4,49,53,54,55,57,60,61,63,66,72,74,81],ear:74,earli:81,eas:43,easi:[52,53,54,61,83],easier:[56,58,60,61,81,83],easiest:63,easili:[3,4],echo:74,edg:74,edit:72,edu:83,effect:[54,61,72,81,82,83],effici:60,efficientnet:82,efficitur:77,eg:85,egesta:77,eget:77,either:[47,48,52,60,61,62,63,69,70,72,74,80],el:65,eleifend:75,element:[57,74,75,78,81],element_typ:44,elementum:77,elit:[75,77],elk:74,els:[43,44,48,70,74,75],elu:65,emb:[33,52,70,75],embed:[52,57,65,70,74,87],embed_engine_in_new_modul:[21,41,45,50,70],emit:53,emphasi:74,empti:[49,66,70,75,80],emum:[16,17],en:72,enabl:[3,4,24,49,52,55,56,58,67,68,70,72,81],enable_precis:61,enabled_precis:[45,49,61,62,69,70,83,85,86,87],enalbed_precis:87,encod:[57,82],encount:63,encourag:85,end:[44,52,60,61,65,70,74,83],end_dim:[61,65],endif:[43,44,45],energi:74,enforc:61,engin:[0,1,17,32,33,37,45,46,48,49,52,53,55,56,58,61,62,64,66,69,70,72,83,84,86,87],engine_converted_from_jit:61,enginecap:[21,38,45,49,50,69,70,86],english:82,enhanc:74,enim:77,ensur:[29,54,55],enter:53,entir:74,entiti:74,entri:[49,60],entropi:[29,30,83],entropy_calibr:68,entropy_calibration_2:[68,83],enumer:[0,1,2,16,17,46,68],environ:[81,85],ep:65,eq:[65,74],equat:74,equival:[32,56,58,60,61,70,80,83],equivil:37,erat:77,erf:65,eric:74,ero:77,error:[16,49,52,53,54,58,61,63,67,70,74,81],essenc:74,essenti:81,est:77,et:77,etc:[72,74,81,87],etiam:77,eu:77,euismod:77,eval:[61,62,85],evalu:[56,57,58],evaluated_value_map:[53,60],even:61,event:48,everi:[55,61,66],everyth:16,ex:[0,1,2,33,46,70,75,77],exact:85,examin:81,exampl:[48,55,57,58,60,61,64,67,69,70,72,73,75,78,80,81,83,84,85],exceedingli:74,except:81,exception_elimin:54,excerpt:75,excit:82,execpt:54,execut:[33,49,52,54,56,57,58,61,63,69,70,80,81,83,85],execute_engin:[57,61],exert:74,exeuct:57,exhaust:61,exist:[4,31,32,37,63,68,69,70,81,82,83],exit:85,exp:65,expand:[54,65],expand_a:65,expect:[48,54,60,61,69,82],experiment:81,explain:81,explan:81,explic:44,explicit:[0,1,2,3,4,45,46,54,64,74,81,83],explicit_batch_dimens:[66,81],explicit_precis:66,explicitli:[55,56,58,62,83,86],explict:44,explictli:0,expon:65,expos:83,express:74,ext:[74,75],extend:[56,58,60,61,65,82],extent:[61,64],extern:[72,74],extra:61,extract:[61,82],extrem:74,ey:74,f16:[52,61,87],f32:52,f:[63,74,80,81],facilisi:77,fact:63,facto:74,factori:[4,29,30,83],fail:[61,87],fake_quantize_per_channel_affin:65,fake_quantize_per_tensor_affin:65,fall:69,fallback:[52,56,58,60,87],fals:[0,1,2,3,4,44,45,46,49,61,65,66,69,70,72,73,74,75,81,83,86],fame:77,familiar:85,far:[74,81],fashion:[61,82],fast:[49,52,70],faster:82,faucibu:77,fc1:[61,80],fc2:[61,80],fc3:[61,80],fc:[49,52,54],feat:[61,80],featur:[52,55,61,81,82,83,86],fed:[3,4,48],feed:[29,30,61],feedforward:82,feel:64,feli:77,feugiat:[75,77],few:[63,69,81],field:[3,4,66,83],fifth:75,figur:[55,75,77],file:[0,1,2,3,4,5,6,7,8,9,10,11,12,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,49,52,55,57,58,61,63,68,69,70,72,73,75,79,81,83,85],file_path:52,find:[4,61,63,81],finder:63,finibu:77,finish:81,first:[48,53,54,61,62,74,75,81,83,85],firstli:85,fit:74,fix:[49,74,81,87],fixed_s:[45,49],flag:[52,55,56,58,62,63,68,84],flaten:49,flatten:[45,47,61,65,80],flatten_convert:61,flesh:85,float16:[52,69],float32:[48,49,52,69,70,81],float64:70,float_int:65,floor:65,floor_divid:65,floordiv:65,flow:[60,74,80,81,82],flox:74,flush:74,fly:80,fold:75,folder:81,follow:[33,52,55,57,61,63,70,72,74,75,79,80,81,82,83,84,85],foo:[74,75,81],foo_kwarg:81,foo_nod:81,forc:[52,70,72,81],force_fp32_output:81,forced_fallback_op:55,form:[53,69,74,85],format:[33,45,48,49,52,62,65,69,70,74,75,82,85],forth:75,forum:63,forward:[29,30,32,33,55,57,60,61,69,70,80,83,86],found:[42,43,44,45,61,63,74,83,84],four:[74,75],fp16:[0,48,49,52,61,62,64,87],fp32:[0,48,49,52,64,70,81,82,83,85],frac:74,freed:60,freeze_modul:54,friend:45,fringilla:77,from:[0,1,2,3,4,29,30,44,46,48,49,52,53,54,55,56,57,58,60,61,64,66,70,72,73,74,75,80,81,82,83,85],from_tensor:81,frontend:[62,64],fssl:63,fstream:[20,44],full:[45,49,52,60,61,67,83,84,85,87],fulli:[31,52,54,61,70,83,87],further:81,fusc:77,fuse_addmm_branch:54,fuse_flatten_linear:54,fuse_linear:54,fusion:[60,81],futur:[70,81],fx2trt_exampl:81,fx:[62,64,69],g:[29,30,52,54,63,66,69,74,81,83],g_:74,gamma:65,gatewai:73,gaurd:43,gcc:[58,61],ge:65,gear:83,gener:[3,4,29,52,54,57,58,60,61,63,66,72,74,75,78,80,81,83],get:[0,1,2,3,4,23,34,44,46,54,55,60,61,63,67,69,81,82,83,85],get_batch:68,get_batch_impl:44,get_batch_s:68,get_build_info:[21,38,45,50,69],get_cache_mode_batch:68,get_is_colored_output_on:[18,39,42,50,67],get_logging_prefix:[18,39,42,50,67],get_output:81,get_reportable_log_level:[18,39,42,50,67],getattr:[54,57,61,80],getbatch:[3,4,44],getbatchs:[3,4,44],getdimens:[60,61],getoutput:[60,61],git:78,github:[59,61,63,72,83,84,85],github_url:72,gitlab:72,gitlab_url:72,give:[72,74,81],given:[48,49,52,54,61,62,66,68,69,70,80,81,86],global:[26,52,61],gnu:63,go:[44,54,55,61,64,80,81,82,85],goal:60,goe:[74,81],good:[44,60,74,81],goodger:75,googl:72,got:[61,74],gpu:[1,32,35,37,45,46,52,61,69,70,81,83,85,86,87],gpu_id:[35,45,46,52,69,83,86,87],graph:[16,31,32,37,45,52,53,55,56,58,60,61,64,67,70,80,81,82],graph_input:[45,49],graph_modul:66,graphinput:[21,38,45,49,50],graphinputsstruct:50,graphmodul:[62,66],gravida:77,great:[61,74],greater:67,group:[65,74,75],grpc:85,gru_cel:65,gt:65,gtc:64,guangzhou:75,guard:54,guard_elimin:54,gui:74,guid:[64,73],gulf:85,gz:[74,75,83],h:[0,1,2,3,4,5,6,7,8,9,10,11,12,15,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,49,50,51,52,54,61,83],ha:[49,53,54,55,56,58,60,61,66,74,75,80,81,82,83],habit:77,habitass:77,hac:77,hack:44,hakaimagazin:85,half:[52,61,62,69,74,83,85,86,87],hand:85,handl:[54,57,81],happen:[80,81],hardtanh:[60,65],hardtanh_:65,hardwar:87,has_batch_dim:66,hash:63,have:[29,33,44,52,53,54,60,61,63,64,66,69,70,74,80,81,82,83,85],haven:61,header:[61,72,74,75,85],heart:75,heaven:74,heck:74,heh:75,hehe:75,height:74,help:[27,52,53,60,61,81,82,84],helper:60,hendrerit:77,here:[44,53,55,57,61,63,72,74,75,80,81,83,84,85],hermet:63,hexagram:74,hfile:50,hi:[65,74,75],hidden:[43,72],high:[54,55,72],higher:[54,72,74,80],highli:[82,85],highlight:74,hinton:83,hold:[46,47,48,53,60,83],holder:[57,76],holi:74,home:63,hood:58,hope:75,host:[49,52,63,70,85],how:[3,4,63,74,76,78,80,82,84,85,86],howev:[29,63,72,73,85],html:[59,63,74,80,83],html_theme:79,html_theme_opt:72,html_theme_path:79,http:[59,61,63,72,74,80,82,83,84,85],http_archiv:63,httpclient:85,hub:85,huggingfac:82,human:74,humankind:75,hx:65,hybrid:70,hyperlink:74,hyphen:74,i8:52,i:[52,54,60,61,74,75,80,83],iaculi:77,icon:[72,74],id:[35,45,52,69,72,73,77,87],idea:[54,74],ident:52,idx:65,ifndef:[44,45],ifstream:44,ignor:69,iii:75,iint8calibr:[3,4,29,30,44,45,49,70,83],iint8entropycalibrator2:[3,4,29,30,44,83],iint8minmaxcalibr:[29,30,83],ilay:60,illustr:[81,82],imag:[83,85],imagenet:82,imagenett:82,images_:83,img1:85,img:85,img_path:85,imperdiet:77,implement:[3,4,54,55,57,61,73,81,83,84],implic:54,implicit:[65,74,81],implicitli:69,implictli:69,improv:75,in_shap:61,in_tensor:80,incas:44,includ:[13,15,16,34,36,42,43,44,45,51,52,55,56,57,58,61,63,66,72,74,80,81,83],includedirectori:50,includehidden:72,incompat:63,incorpor:75,indent:74,index:[33,59,64,65,70,72,78,83],indic:[65,72,74],indirect:74,inetworkdefinit:53,infer:[54,61,69,70,81,82,83],inference_output:85,inferenceservercli:85,inferinput:85,inferrequestedoutput:85,info:[16,32,37,45,52,60,61,67,69],inform:[25,33,34,36,48,52,53,55,57,61,63,64,66,67,69,74,80,81,83,86],infrastructur:[83,85],ingest:58,inherit:[50,81,83],initi:74,injuri:74,inlin:[0,1,2,3,4,29,30,44,46,48,54,61,75,78],inner:[49,75,82],input0:61,input1:61,input2:61,input:[3,4,21,29,33,38,44,45,47,49,50,52,53,54,55,57,60,61,62,65,66,67,69,70,75,80,81,82,83,85,86,87],input_0:[57,61],input__0:85,input_data:[62,80],input_file_path:[52,87],input_is_dynam:45,input_nam:[66,81],input_s:[55,61],input_scal:65,input_shap:[83,87],input_signatur:[45,47,49,70],input_spec:[52,66,81],input_tensor_spec:[66,81],input_v:81,inputclass:50,inputrang:[55,61],inputtensorspec:[66,81],insert:[61,83],inserting_befor:81,insid:[74,85],inspect:[60,61,80],instal:[61,64,78,84,85],instanc:[54,61,68,80,82],instance_norm:65,instanti:[56,57,58,60,61],instatin:[0,1,2,46],instead:[49,52,53,54,61,63,84],instnanti:57,instruct:[55,56,58,61,63,81,85],insur:63,int32:[69,70,82],int64:70,int64_t:[45,46,48,49,83,87],int8:[0,44,48,49,52,64,69,70,83,87],int8_t:[17,45],int8cachecalibr:[20,29,40,44,50],int8cachecalibratortempl:50,int8calibr:[3,20,30,40,44,50],int8calibratornamespac:50,int_float:65,integ:[69,77],integr:64,intend:63,intent:[54,74],interact:74,interdum:77,interest:[54,74],interfac:[0,1,2,46,57,58,60,83],interfer:74,intermedi:[16,49,52,67,70,80],intern:[1,16,46,60,61,67,74],internal_error:67,internalerror:67,interpol:74,interpret:[57,74,81],intro_to_torchscript_tutori:80,introduc:[81,82],invok:[61,80,81],io:[44,85],iostream:[20,21,44,45,61],ipso:74,ipsum:[75,77],ir:[56,58,60,62,69,80],is_floating_point:65,is_train:83,iscustomclass:60,isinst:81,isn:[72,74],issu:[3,4,61,63],istensor:60,istream_iter:44,it_:44,ital:74,item:[73,75],itensor:[53,60,61,81],iter:[20,44,49,52,53,68,70],its:[29,53,57,60,63,74],itself:[0,1,2,46,52,54,63,85,86],iv:75,ivalu:[45,47,49,53,57,60,61],jan:75,jetpack:63,jetpack_5:63,jetpack_x:63,jetson:[69,82],jit:[31,32,33,37,45,47,49,52,53,54,55,56,57,58,59,60,61,62,69,70,80,85,86],jp_workspac:63,jpg:85,jump:85,just:[44,45,54,61,62,64,67,74,76,80,81,82,84,86],justo:[75,77],k:[65,83],kbool:[0,45],kchannelslast:[2,45],kchar:[0,45],kclip:60,kcontigu:[2,45,48],kcpu:[1,46],kcuda:[1,46,55,61],kdebug:[16,42,44],kdla:[1,45,46,87],kdla_standalon:[17,45],keepdim:65,kei:[74,80,85],kept:75,kernel:[48,49,52,60,69,70,81],kernel_s:65,kerror:[16,42],keyboard:74,keyword:[69,70],kf16:[83,87],kfloat:[0,45,49],kgpu:[1,45,46],kgraph:[16,42,54],khalf:[0,45,61],ki8:83,kind:[53,69,81],kinfo:[16,42,44],kint:[0,45],kinternal_error:[16,42],know:[42,60,72,74],knowledg:74,kriz:83,krizhevski:83,ksafeti:[17,45],kstandard:[17,45,49],ktest:83,ktrain:83,kunknown:[0,2,45],kwarg:[68,69,81,82],kwarn:[16,42],l:65,label:[74,82,83,85],lacinia:77,lack:[55,56,58,81],lacu:77,laid:61,lambda:[60,61,74,85],lang:73,languag:[73,74,75,80,85],laoreet:77,larg:[56,58,61,72,74,82,83],larger:[72,82],largest:65,last:[2,54,69,81],lastli:85,later:[29,61],latest:[63,72],launch:85,layer:[46,49,52,53,54,60,61,70,81,82,83,85,87],layer_norm:65,layout:[2,48,65,69,70],ld_library_path:63,ld_preload:84,ldd:63,le:65,lead:74,leader:74,leaky_relu:65,leaky_relu_:65,learn:[61,64,83,85,87],leas:75,least:[74,75],leav:54,lectu:[75,77],left:[72,74],legacy_calibr:68,legend:74,len:65,lenet:[61,80],lenet_script:[61,80],lenetclassifi:80,lenetfeatextractor:80,length:[3,4,44,65,75,81],leo:77,let:[46,52,54,60,69,70,72,74,81,82,85],letter:[75,82],level:[18,23,25,26,39,42,44,50,54,55,58,67,70,78,80,81,85],levelnamespac:50,leverag:[81,83],lib:[54,61,63],libero:[75,77],librari:[34,42,43,44,45,52,56,57,58,60,61],libtorch:[4,36,60,61,63,83],libtorch_pre_cxx11_abi:63,libtorchtrt:[52,61,63],libtorchtrt_plugin:84,libtorchtrt_runtim:84,licens:[42,43,44,45,61],light:74,ligula:77,like:[52,53,54,57,60,61,62,63,73,74,80,81,83,84,85],limit:[54,67,73,83],line:[52,61,75],linear:[2,65,69,80],link:[52,53,61,64,72,73,78,84],linux:[58,61,63],list:[18,19,20,21,31,49,51,53,55,57,60,61,62,63,65,66,69,70,78,81,85],listconstruct:[53,57,61],listunpack:[57,61],liter:75,literal:75,literal_block:74,live:[60,74],ll:81,lo:65,load:[52,55,57,61,62,68,70,81,82,83,84,85,86],load_librari:84,loading_data_recip:83,loborti:[75,77],local:[52,54,61,72],localhost:85,locat:[63,83],lock:73,log:[15,16,19,20,38,44,50,51,54,60,64,65,69,81],log_debug:60,logger:67,logger_level:66,loggingenum:50,logic:81,login:85,logist:81,loglevel:67,logo_onli:72,lone:75,longer:[72,84],look:[53,54,80,83,85,86],loop:81,loop_unrol:54,lorem:[75,77],lose:72,loss:[82,83],lot:60,low:81,lower:[16,67,69,75,82],lower_exampl:81,lower_graph:54,lower_precis:81,lower_tupl:54,loweralltupl:54,lowersimpletupl:54,lstm_cell:65,lt:65,ltorchtrt:84,luctu:77,lvl:[25,26,42],m:75,machin:[57,83,85],macro:[5,6,7,8,9,10,11,12,15,18,20,21,42,44,45,50,51],mad:74,made:[54,56,58,74],maecena:77,magna:77,mai:[53,57,58,61,62,70,74,75,80,81,83,85],main:[54,55,56,57,58,60,61,72,74,76,81],mainli:81,maintain:[55,57,60],major:[58,81],make:[53,61,62,63,74,76,81,82,83,85,87],make_data_load:[4,83],make_int8_cache_calibr:[20,40,44,50,83],make_int8_calibr:[20,29,40,44,50,83],malesuada:77,man:[74,75],manag:[49,52,53,56,58,60,61,67,69,70],mangag:54,mani:[72,74,75,81],mantissa:[49,70],manual:[73,74,81],map:[1,46,53,54,56,58,60,61,81,82,83,85,86],mapper:81,mark:[54,72],marknodesforfallback:54,markup:[75,78],markup_process:74,mask:65,masked_fil:65,massa:77,master:[59,63,74,83,84],mat2:65,match:[54,63],math:78,matmul:[54,61,65],matrix:59,matter:81,matti:75,matur:58,mauri:[75,77],max:[48,52,60,65,69,72],max_batch_s:[81,85],max_c:52,max_h:52,max_input_shap:66,max_n:52,max_pool1d:65,max_pool2d:[61,65,80],max_pool3d:65,max_shap:[45,48,62,69,70,81,82],max_val:[60,65],max_w:52,max_workspace_s:81,maximu:77,maximum:[48,49,52,70,81,85],mayb:74,mb:52,md:59,me:[74,75],mean:[55,60,64,65,81,85],mechan:[60,81,82],medium:74,meet:69,member:[46,47,48,49,69],memeori:2,memori:[20,21,44,45,54,60,61,62,69,70],memory_format:[65,69],memoryformat:[2,45],men:74,mental:74,menu:[52,72,74],menuselect:74,messag:[16,25,26,52,67],meta:[78,81],metadata:[49,52,57,60,70,72],meth:74,method:[31,32,33,37,48,52,54,60,61,63,69,70,74,80,82,86],method_nam:[31,37,45,52,61,69,70],metu:77,mi:77,middl:74,might:[54,63,72],min:[48,52,60,65,69],min_block_s:[45,49,55,70],min_c:52,min_h:52,min_input_shap:66,min_n:52,min_shap:[45,48,62,69,70,81,82],min_val:[60,65],min_w:52,mind:74,mine:74,minim:83,minimum:[48,49,52,55,67,70],minmax:[29,30,83],minmax_calibr:68,misbuild:72,miss:[61,74],mkdir:63,mm:85,mmb:74,mobilenet_v2:86,mobilenetv2:82,mod:[52,55,61,78,81,83],mode:[62,81,83],mode_:83,model:[52,55,57,61,62,64,66,67,80,83,86],model_nam:85,model_repositori:85,model_torchtrt:67,model_trt:67,modifi:[75,81],modul:[31,32,33,37,45,49,52,55,56,57,58,60,62,63,64,66,69,70,73,74,75,81,82,83,86,87],modular:61,module_fallback:54,module_nam:52,molesti:77,momentum:65,morbi:77,more:[53,61,63,64,69,72,75,80,81,83,84,85,86],most:[58,63,66,81,84,85],mother:74,motion:74,mous:74,move:[30,44,54,57,61,70,83],msg:[26,42,67],mu:74,much:[60,72,74,83],mul:65,mul_:65,multi:52,multipl:[57,74,75,83,85],multipli:[49,70],must:[33,48,49,52,54,55,60,61,63,69,70,74,75,81,84],mutil:75,my:74,my_pytorch_model:81,myclass:74,mymodel:[55,62],myself:75,n:[52,60,61,83],nabla:74,nam:[75,77],name:[3,4,31,33,37,44,55,57,60,61,63,68,70,74,75,80,81,85,86],namedtupl:81,namespac:[42,43,44,45,51,54,64,83],narrow:65,nativ:[58,59,61],native_funct:59,natur:74,nav:[72,78],navig:72,navigation_depth:72,nbbind:[3,4,44],nchw:[2,69,70],ne:[54,65],nec:77,necessari:[42,84],need:[0,1,2,25,29,43,46,53,54,60,61,62,63,66,74,81,82,83,84,85],neg:65,negative_slop:65,nequ:[75,77],nest:[45,49,50,74,75],net:[60,61,74,75],netu:77,network:[29,30,60,61,81,82,83,85,87],neural:87,new_lay:60,new_local_repositori:63,new_siz:83,newer:63,next:[3,4,53,57,72,74,75,83,85],ngc:[63,85],nhwc:[2,52,69],nibh:[75,77],nice:63,nickel:74,night:75,nightli:81,ninja:63,nisi:77,nisl:77,nlp:[29,30,83],nn:[54,59,61,62,69,70,80,81],node:[54,55,56,58,60,61,81,82],node_info:[60,61],noexcept:[3,4,44,83],non:[75,77],non_block:65,none:[60,65,66,69,70,72,74,81],nonetheless:74,nonexist:74,norm:65,normal:[0,1,2,46,61,74,80,81,83,85,87],normalized_shap:65,noskipw:44,notatemoduleforfallback:54,note:[1,46,48,60,61,63,69,72,74,81,87],notebook:[58,64],now:[54,58,60,61,63,74,81,86],np:85,nu:74,nulla:77,nullptr:[44,45,49],num:52,num_avg_timing_it:[45,49,70,86],num_it:52,num_op:52,num_work:83,number:[3,4,49,52,54,55,60,61,62,69,70,72,81,82],numel:65,numer:[52,75,81],numpi:85,nunc:77,nvcr:85,nvidia:[32,37,42,43,44,45,52,59,61,63,69,70,81,85,87],nvinfer1:[3,4,29,30,44,45,49,60,83],nvinfer:[20,44],o:[63,74,85],obj:65,object:[0,1,2,3,4,46,48,49,52,57,60,67,68,70,83,86],obtain:82,obvious:80,odio:[75,77],off:[55,57],offici:63,ofstream:[44,61],often:74,oh:75,ok:[61,74],okai:49,older:58,onc:[42,43,44,45,53,54,57,81,83,84,85],one:[47,54,60,61,62,67,69,74,80,81,85],ones:[42,55,56,58,61,63,74],onli:[1,3,4,16,29,44,46,48,52,54,55,58,60,63,66,67,69,74,81,83,84,87],onnx:54,onto:[52,57],op:[52,53,54,56,58,60,61,69,84],op_and_target:81,op_nam:52,open:[82,85],oper:[0,1,2,3,4,31,44,45,46,49,52,53,54,55,56,57,58,60,62,64,69,70,81,83,87],oppos:70,opset:[56,58],opt:[48,63,69],opt_c:52,opt_h:52,opt_n:52,opt_shap:[45,48,62,69,70,82],opt_w:52,optim:[48,52,54,61,62,64,66,80,81,82],optimin:48,optimiz:80,optimize_target_shap:81,optimized_input_shap:66,optimz:85,option:[44,48,52,55,56,58,63,69,74,78,81,83,84,87],orchestra:74,orci:77,order:[49,55,60,61,62,63,66,70,81],org:[59,61,63,72,74,80,83],organ:75,origin:81,ornar:[75,77],os:45,ostream:45,other:[0,1,2,45,46,52,53,54,57,61,62,63,64,65,73,74,81,84],otherwis:[63,84],our:[55,58,61,80,85],out:[31,44,53,54,55,56,58,60,61,63,67,70,74,85],out_shap:61,out_tensor:[60,61],output0:54,output:[24,27,33,49,52,53,54,55,57,60,61,63,67,70,72,74,75,81,82,85],output__0:85,output_file_path:[52,87],output_nam:[66,81],output_pad:65,output_s:65,outself:61,outsid:74,over:[56,58,74,81,85],overal:82,overrid:[3,4,29,30,44,69,81,83],overview:[59,64],own:[60,61,63,74,85],p:[52,61,65,85,87],packag:[52,54,61],pad:65,padding_idx:65,page:[64,76,78,85],pair:[54,60,63,74,82,83],pane:74,paragraph:[75,78],param:[68,73],paramet:[0,1,2,3,4,25,26,27,29,30,31,32,33,35,37,46,48,49,53,54,60,61,67,69,70,78,80,81],parent:[14,15,18,19,20,21],pars:[61,74],parser:74,part:[52,55,58,72,73,74,81],partial:[52,74],partit:54,partitioninfo:55,pass:[53,55,56,57,58,60,61,63,67,68,80,81,83],past:74,path:[4,13,14,15,29,30,52,61,63,68,69,80,81,83,85],path_to_torchtrt_root:63,pathwai:80,pattern:[60,61,69],payment:73,pbtxt:85,peephole_optimz:54,pellentesqu:77,peopl:74,pep:74,perforamnc:81,perform:[29,30,82,83,85],permit:74,permut:[65,81],persist:74,pharetra:77,phase:[16,60,61],phasellu:77,phi:74,philosoph:74,phrase:74,pi:74,pick:80,pickler:57,piec:82,pil:85,pin:73,pin_memori:65,pip3:63,pip:[63,85],pipelin:[52,87],piplein:61,pixel_shuffl:65,pl:73,place:[48,54,63,74,75,76,81,83],placerat:77,plan:[52,58],platea:77,platform:[45,52,58,63,85,87],pleas:[61,63,74,81,85],plugin:81,point:[61,69,72,73,74,85],pointer:[3,4,83],polish:73,pool:87,pop:57,popul:66,popular:[63,73,82],portabl:[57,70],portion:74,porttitor:[75,77],posit:[52,69,72,81],possibl:[63,74,82,85],post:[29,30,49,52,61,64],posuer:[75,77],potenti:[49,77],pow:65,power:[61,74,81,82],pr:61,praesent:77,pragma:[42,43,44,45,83],pre:[33,54,68,70,83,84],pre_cxx11_abi:63,preced:74,precis:[49,52,61,62,64,69,81,83,87],prefer:61,prefix:[27,28,42,67,74],preinstal:63,prelu:65,prepar:[81,85],preprint:83,preproc:68,preprocess:[83,85],preserv:[74,80,83],prespect:80,press:74,pretium:77,pretrain:[82,85,86],pretti:61,prev_next_buttons_loc:72,prevent:[49,52],previou:72,previous:[29,33,61],prim:[53,54,57,61,65,80],prim_devic:65,primal:74,primarili:[58,61],print:[16,31,44,61,67,69,70,74,85,86],priorit:63,privat:[3,4,44,45,83],problem:74,problemat:74,proce:85,proceed:85,process:[52,55,73,74,80,82,83,85,86],prod:65,produc:[48,53,57,60,61,74,82],product:49,profil:[48,66],profiling_verbos:81,program:[18,19,20,21,29,51,52,56,57,58,64,80],programm:74,progress:75,proin:77,project:[63,73,78],promis:81,prop:66,properli:63,properti:72,propog:54,prose:74,provid:[3,4,49,52,55,57,60,61,62,63,66,69,70,74,81,83,84,85,86],providi:[56,58],provok:74,pt:[52,61,81,85],ptq:[3,4,15,18,19,38,50,51,52,64,69,70],ptq_calibr:[3,4,45,49,83],ptqtemplat:50,publish:85,pull:[63,85],purchas:73,pure:31,purpos:[63,81,82,85],puru:77,push:57,push_back:[44,55],put:[74,82],pwd:85,py3:85,py:[54,58,61,63,72,74,79,80,81,83],pyindex:85,pypi:63,python3:[54,61,63],python:[52,55,58,61,69,70,74,75,81,82,84,85,86,87],python_api:59,pytorch:[48,49,52,54,55,56,57,58,60,61,62,63,68,69,70,80,83,84,85],pytorch_libtorch:85,pytorch_sphinx_them:[72,79],qat:82,quant_max:65,quant_min:65,quantiz:[29,30,52,61,64],quantizatiom:49,quartznet:82,question:61,qui:[75,77],quickli:[52,61,83],quisqu:77,quit:[60,61,82],quot:75,r:74,rais:[54,81],raiseexcept:54,ram:[49,52,70],rand:[61,81],randn:[55,61,69,70,86],rang:[48,49,52,69,81,82],rank:72,rather:54,raw:72,re:[74,81],read:[3,4,29,30,44,72,74,83],read_calibration_cach:68,readcalibrationcach:[3,4,44],reader:74,realiz:57,realli:60,reason:[0,80,81],reattribut:75,recalibr:29,receiv:81,recip:83,reciproc:65,recognit:[82,83],recomend:[29,30],recommend:[29,30,61,63,74,81,85],record:[53,80],recurs:53,redistribut:75,reduc:[54,56,58,81,82,83],redund:81,ref:74,refer:[48,56,58,61,73,78,81,83,85],referenc:63,refit:[45,49,70,86],reflect:45,reflection_pad1d:65,reflection_pad2d:65,regard:[63,74],regardless:75,region:81,regist:[33,57,60,70,81],register_acc_op:81,register_acc_op_map:81,register_custom_acc_mapper_fn:81,registernodeconversionpattern:[60,61],registr:81,registri:[53,61],reinterpret_cast:44,rel:52,relat:[46,74],relationship:50,releas:74,reload_model_output:81,reload_trt_mod:81,relu:[55,61,65,80],relu_:65,remain:[54,83],rememb:81,remov:72,remove_contigu:54,remove_dropout:54,remove_to:54,render:72,rent:75,repack:57,repeat:[52,65],replac:[54,63],replication_pad1d:65,replication_pad2d:65,replication_pad3d:65,report:[23,44],reportable_log_level:67,repositori:[58,72,79,85],repres:[48,49,60,67,74,81],represent:[54,60,80,81,82],request:[61,69,85],requir:[29,49,52,53,54,61,67,69,70,72,81,83,84,85],require_full_compil:[45,49,70],requires_grad:65,research:81,reserv:[42,43,44,45],reset:44,reshap:[65,85],resiz:85,resnet50:85,resnet:[57,82,85],resnet_trt:57,resolut:82,resolv:[53,54,56,58],resourc:[53,83],respons:[29,57,74],rest:[74,75,81],restrict:[49,70],restructuredtext:[74,75],result:[53,54,62,67,70,72,80,85],ret:54,reus:[54,81,83],revert:72,revis:[74,75],revisit:74,rfc:74,rho_:74,rhoncu:77,right:[42,43,44,45,54,58,60,74],risu:77,rm:85,rn50_preprocess:85,role:74,roll:65,roman:75,room:74,root:[42,43,44,45,63,72,83],roughli:55,round:[49,70],rounding_mod:65,row:75,rst:[72,74],rsub:65,rtol:52,rule:[63,70,81],ruler:74,run:[1,37,46,49,52,53,54,55,56,57,58,60,61,62,63,64,66,69,70,74,80,81,82,83,84,85,86,87],running_mean:65,running_var:65,runtim:[61,64],runtimeerror:81,rutrum:[75,77],s:[48,49,55,57,60,61,62,63,64,66,69,72,74,75,80,81,82,83,85],safe:[60,70],safe_dla:69,safe_gpu:69,safeti:[49,52,69],sage:74,sagitti:[75,77],sai:[75,82],said:74,same:[57,61,63,72,74,80,81,85,86],sampl:[74,81,83,85],sample_input:81,sapien:77,satisfi:[55,81],save:[29,44,52,57,61,62,69,70,81,82,84,85],saw:61,scalar:[60,65],scalaropt_dim:65,scalartyp:[0,45,65],scale:[65,82,83],scale_factor:65,scale_grad_by_freq:65,scales_d:65,scales_h:65,scales_w:65,scelerisqu:77,schedul:[69,85],schema:[60,61],scheme:81,scientist:74,scope:54,scratch:29,scratch_spac:85,screen:72,script:[31,54,55,61,62,69,70,80,86],script_model:[80,86],scriptclass:70,scripted_model:87,scriptmodul:[61,62,69,70],scroll:[72,76],sdk:59,se:82,seamlessli:64,search:[64,72],second:[54,74,81],secondli:85,section:[61,72,74,75,76,78,81,83,85],sed:[75,77],see:[31,54,57,61,63,69,70,74,80,81],seen:[74,75],segment:[55,82],select:[17,29,30,37,49,52,57,62,63,65,69,70,73,76,81,83],self:[54,57,60,61,65,68,80,82,87],self_1:[57,61],self_int:65,sell:75,seller:73,seller_id:73,sem:77,semant:74,semper:77,send:85,senectu:77,sens:[61,74],sentenc:[74,82],sentinel:[0,2],separ:[55,56,58],sequenc:[66,74,81,82],serial:[33,37,52,56,58,61,69,70],seriali:70,serializ:[57,80],serialized_cach:[66,81],serialized_engin:70,seril:57,serv:[52,57,64,81],servic:74,session:74,session_nam:74,set:[3,4,16,21,25,27,29,32,35,37,45,46,48,49,53,54,55,56,57,58,61,62,63,64,66,67,69,70,72,76,79,80,81,82,83,87],set_data_from_numpi:85,set_devic:[21,38,45,50,69],set_is_colored_output_on:[18,39,42,50,67],set_logging_prefix:[18,39,42,50,67],set_reportable_log_level:[18,39,42,50,67],setalpha:60,setbeta:60,setnam:[60,61],setreshapedimens:61,setup:[43,83,85],sever:[16,26,67],sh:63,sha256:63,shape:[45,47,48,49,52,55,60,62,65,66,69,70,81,85,87],shape_mod:69,shape_rang:[66,81],share:[49,52,63,70],shell:52,shell_command:74,shift:[63,65,74],ship:[61,84],shorthand:74,should:[0,3,4,29,45,49,52,53,54,55,56,58,60,64,67,69,70,72,74,77,81,83,85],show:[72,74,82],shown:[61,72,74],shuffl:[61,83],side:[54,61,72],sidebar:[72,78],sigmoid:[65,81],sigmoid_:65,sign:85,signatur:70,signifi:[48,54],signific:74,significantli:[54,72],similar:[60,61,81,84,86],simonyan:83,simpil:83,simpl:[74,75,80,81,82,85],simplest:85,simpli:[54,82],simplifi:53,simul:82,sin:[65,74],sinc:[54,61,74,80,81,83],sing:74,singl:[48,52,54,61,69,74,80,81,83],singular:60,sinh:65,sink:74,sit:[75,77],site:[54,61,63,74],six:74,sixth:75,size:[3,4,44,48,49,52,54,55,61,65,69,70,72,81,82,83],size_t:[3,4,44,83],skip:52,slash:72,slice:65,slightli:81,sm:57,small:[54,85],smaller:82,so:[0,44,52,53,54,57,58,60,61,63,64,73,74,75,81,83],sodal:77,softmax:[54,65,81],softwar:[49,52,70,74],sole:[62,83],sollicitudin:77,solv:85,some:[53,54,56,57,58,60,61,73,74,81,83],some_funct:74,someth:[43,54,74,85],someurl:74,sort:[60,65,86],sourc:[42,43,44,45,58,66,67,68,69,70,81],sourceforg:[74,75],space:[74,75,83],spaces_and_linebreak:74,span:75,spars:[52,65],sparse_weight:[45,49,70,81],sparsiti:[49,52,70,81],spec:[45,48,49,52,67,69,70,86],specif:[32,49,54,56,58,69,70,74,82],specifi:[3,4,52,60,62,63,64,67,69,70,72,74,81,85,86],specifii:69,speech:82,speedup:82,sphinx:[72,73,74,75,79],sphinx_rtd_them:[74,75],spin:85,spirit:74,split:[65,81],split_siz:65,split_with_s:65,sqrt:65,squeez:[65,82],sram:52,src:[57,59,65],ss:44,ssd300_trt:57,ssd:57,ssd_trace:52,ssd_trt:52,sstream:[20,44],stabl:[59,70,72],stack:[57,65,83],stage:[53,81],stand:[57,74],standalon:74,standard:[52,57,64,74,82,84,86],stapl:75,start:[53,55,61,63,65,75,81,82,86],start_dim:[61,65],start_step:65,state:[53,60,61],statement:[54,74],static_cast:44,statu:[44,75],std:[3,4,22,26,28,29,30,31,33,34,37,42,44,45,47,48,49,55,61,83,85,87],stdout:[36,67,69],steamlin:83,step:[64,65,81,82,83],stick:72,sticki:[72,78],sticky_navig:[72,76],still:[44,55,81,83],stitch:[55,61],stop:61,storag:83,store:[2,4,49,52,53,57,60,61,70,80,81],str:[19,43,44,50,65,67,69,70,81],straight:60,strang:74,strategi:69,street:75,strict:84,strict_type_constraint:81,stride:65,string:[3,4,18,20,21,22,26,28,29,30,31,33,34,37,42,44,45,49,55,57,60,61,69,72,83],stringstream:44,strip_prefix:63,strong:74,strongli:74,struct:[1,21,38,41,45,83],structur:[29,46,49,55,58,60,72,74,78,80,85],structuredtext:74,stub:75,stuff:74,style:[42,43,44,45,72,74,75],style_external_link:72,sub:[65,74,80],sub_:65,subdirectori:51,subexpress:54,subgraph:[49,52,53,54,60,61],subject:58,submenu:78,submodul:80,subscript:74,subsect:74,subset:[82,83],substitut:74,subtitl:74,subtre:79,subword:82,sudo:63,suffic:54,suggest:85,suit:64,suitabl:81,sum:[49,65,70,81],superscript:74,supervis:82,suppli:74,support:[0,1,2,27,31,46,48,49,52,55,59,61,62,63,64,69,70,72,73,80,81,85,87],sure:[61,62,63,85,87],suscipit:[75,77],suspendiss:77,symbol:[33,63,70,74,81,84],symlink:79,system:[53,60,63,64,70],t1:65,t2:65,t:[0,1,2,45,46,54,60,61,63,65,72,74,75,80,81,83,85],t_:74,tabl:[63,78],tag:[74,85],take:[31,32,33,37,53,56,57,58,60,61,69,70,72,74,81,82,83,86],taken:74,talk:64,tan:65,tanh:65,tanh_:65,tar:[63,74,83],tarbal:[61,83],target:[1,33,45,46,48,49,52,57,58,62,64,69,70,81,83,86,87],targets_:83,task:[29,30,81,82,83],techinqu:61,techniqu:83,tell:[54,55,56,57,58,60,74],tellu:77,tem:52,templat:[20,40,44,45,50,61,72],temporari:81,tempu:77,tensor:[2,33,44,45,48,49,52,53,54,55,57,60,61,65,66,69,70,80,81,82,83],tensor_mod:65,tensor_scalar:65,tensor_tensor:65,tensorcontain:60,tensorformat:[21,38,45,48,50,69],tensorformatenum:50,tensorlist:[55,60],tensorrt:[0,1,3,4,29,30,31,32,33,36,37,44,45,46,48,49,52,53,54,55,56,58,60,66,68,69,70,80,83],tensorrt_convert:81,tensorrtcompilespec:[70,86],tensort:81,teo:52,term:[69,74,75,82,83],termin:[27,52,61],test:[52,58,63,74,75,81,82,83,85],test_acc_trac:81,test_ptq_dataloader_calibr:83,test_ptq_trt_calibr:83,test_py_modul:[74,78],testing_dataload:83,testing_dataset:83,text:[67,75,77,82],tf32:[49,52],than:[54,64,73,74,82,84],thats:[53,83],the_model_repositori:85,thei:[46,52,53,54,57,60,62,63,69,72,74,81],them:[54,55,57,61,63,72,81,82],theori:[53,74],therebi:[57,82],therefor:[29,57,61,74,81,82],theres:84,therfor:84,theta:74,thi:[0,1,2,29,30,42,43,44,45,46,47,48,49,52,53,54,55,56,57,58,60,61,63,66,69,70,72,73,74,76,77,80,81,82,83,84,85,86],thicker:74,thin:74,thing1:74,thing2:74,thing3:74,thing:[63,74,81],think:[60,74],third:[75,81],third_parti:[58,63],this_arg_is_opt:81,those:[53,74],though:[52,58,60,61,80],thought:74,three:[48,56,58,66,69,74,75,81,82,85],threshold:52,through:[48,53,54,55,57,61,62,64,67,68,74,81,82],throught:81,thrown:[49,70],thu:74,time:[49,52,53,54,56,57,58,60,61,70,72,74,81,83],timing_cach:81,tincidunt:77,tini:83,titles_onli:72,tmp:61,toctre:72,tocustomclass:60,todim:61,todo:[72,81],togeth:[53,60,61],token:82,toler:52,too:[63,72,74,75],tool:[60,61,81,82],toolchain:[58,63],top:[58,72,76],topk:65,torch:[0,1,2,4,20,21,29,30,31,32,33,36,37,44,45,46,47,48,49,52,53,54,55,56,57,58,60,63,66,69,70,80,83,87],torch_executed_modul:[45,49,55,70],torch_executed_op:[45,49,55,70],torch_scirpt_modul:80,torch_script_modul:61,torch_tensorrt:[0,1,2,3,4,14,16,17,42,43,44,46,47,48,49,50,51,52,55,61,62,64,81,82,83,84,85,86,87],torch_tensorrt_export:43,torch_tensorrt_major_vers:[19,43,50],torch_tensorrt_minor_vers:[19,43,50],torch_tensorrt_patch_vers:[19,43,50],torch_tensorrt_vers:[19,43,50],torch_tensorrtfil:50,torch_tensorrtnamespac:50,torchbind:57,torchhub:85,torchscript:[19,21,38,43,45,49,50,52,56,57,58,62,69,70,82,86,87],torchscriptstruct:50,torchtrt:[43,55],torchtrt_api:[0,2,19,22,23,24,25,26,27,28,31,32,33,34,35,36,37,42,43,44,45,48,49,50],torchtrt_check:60,torchtrt_hidden:[19,43,50],torchtrt_runtime_exampl:84,torchtrt_unus:60,torchtrtc:[63,64,87],torchvis:[57,83,85,86],toronto:83,tortor:77,totensor:[83,85],tovec:61,toward:83,trace:[55,61,70,80,81],traced_model:80,track:[60,83],tradit:[48,70,83],traget:32,trail:72,train:[29,30,49,52,61,62,64,65],trainabl:54,transcrib:82,transfer:73,transform:[61,83,85],transformed_img:85,translat:61,transmit:74,transpos:[65,81],trash:74,travers:[56,58],treat:52,tree:[42,43,44,45,72,83,84],trigger:[61,81],trim:83,tristiqu:77,triton:64,triton_to_np_dtyp:85,tritoncli:85,tritonserv:85,trt:[0,1,3,4,46,48,53,54,57,60,61,65,81],trt_interpreter_result:81,trt_lenet_script:61,trt_mod:[55,61,83,87],trt_model:[55,85,86],trt_ts_modul:[55,62],trtinterpret:[66,81],trtinterpreterresult:[66,81],trtmodul:[66,81],truncat:[49,52,70],truncate_long_and_doubl:[45,49,70],ts:[43,52,55,61,62,64,69,80,86],ts_model:[55,61],tt:74,tue:75,tupl:[57,66,69,70,81],tupleconstruct:[54,57],tupleunpack:54,turpi:77,tutori:[80,83],two:[52,54,60,62,63,74,75,79,80,81,83,85],type:[0,1,2,30,49,50,52,53,57,60,61,62,66,67,69,70,74,81,82,83],type_fp32:85,typenam:[3,4,29,30,44],typic:[53,60,85],ugli:74,ui:73,uint64_t:[45,49],ultric:77,un:83,unabl:[60,61],unbind:65,unbroken:74,uncas:82,uncom:63,under:[42,43,44,45,58,74],underli:[0,1,2,46,60],uniformli:82,union:[60,61,69,70],uniqu:[4,62],unique_ptr:[4,30],unit:81,univers:74,unknown:69,unless:81,unlik:[63,64,86],unlimit:72,unpack_addmm:54,unpack_log_softmax:54,unqiue_ptr:4,unreferenc:74,unrestrict:74,unsqueez:65,unstabl:58,unsupport:[31,49],unsur:60,untest:58,until:[53,58,60,63],unwrap:60,unwraptodoubl:60,unwraptoint:61,unzip:63,up:[53,54,56,57,58,74,80,81,82],updat:81,upload:85,upon:72,upper:75,upsample_bilinear2d:65,upsample_linear1d:65,upsample_nearest1d:65,upsample_nearest2d:65,upsample_nearest3d:65,upsample_trilinear3d:65,upscale_factor:65,upstream:61,uri:74,url:[63,72,85],urna:77,us:[0,1,2,3,4,29,30,32,35,37,43,44,45,46,48,49,52,53,55,57,58,60,64,66,67,68,69,70,72,73,74,75,80,81,83,84,85,87],usag:[61,68,74],use_cach:[3,4,30,44,68,83],use_cache_:44,use_cmake_generated_export_head:43,use_input_stat:65,use_subset:83,usecas:[62,63],user:[42,48,55,56,57,58,61,62,63,64,74,75,83,85],using_int:[61,65],usr:63,usual:[72,81],ut:77,utf:[74,75],util:[60,61,70,82,83,85],v0:[71,85],v2:[29,30],v:[52,75,85],valid:[1,46,60],valu:[0,1,2,16,17,45,46,48,53,57,60,61,65,67,68,69,72,82],value_tensor_map:[53,60],vanilla:81,vari:66,variabl:[48,69,81],variant:84,varient:54,varieti:85,variou:[81,87],variu:77,vcs_pageview_mod:72,vec:65,vector:[20,21,44,45,47,48,49,55,57,61,83,87],vehicula:77,vel:77,velit:77,venenati:77,verbios:52,verbos:[52,75],veri:[75,76,81,83,85,86],verifi:55,verison:63,version:[34,36,58,63,72,75,81,82,85],vertic:[72,74],vestibulum:[75,77],vgg16:83,vgg:83,vi:74,via:[64,69,70,72,78,81,82,83,84],view:[65,72],virtual:83,vision:[81,85],visitor:72,vita:[75,77],vivamu:77,viverra:77,vm:75,volutpat:77,vs:[0,1,2,46,54,70,86],vulput:77,w:52,w_hh:65,w_ih:65,wa:[54,57,61,74,81],wai:[52,61,63,80,81,82,83],walkthrough:82,want:[42,55,61,66,80,81,83,85,86],warn:[16,44,52,60,67],wash:74,we:[42,44,53,54,56,57,58,60,61,66,72,74,80,81,82,83,85],weak:74,web:74,websit:63,weight:[48,49,52,53,61,65,70,74,81,82],welcom:[61,81],well:[61,63,67,74,83],were:61,wget:85,what:[4,54,61,62,74,80,81],whatev:[57,81],wheel:63,when:[27,44,45,46,52,53,54,56,57,58,60,61,63,67,69,70,72,74,76,80,81,82,83],where:[53,54,60,61,70,75,81,83],wherev:81,whether:[4,52,66,69,73,81,83],which:[1,2,29,32,37,46,49,53,54,55,56,57,58,60,61,62,63,68,70,72,74,75,80,81,82,83,84,85,86],white:74,whitespac:74,whl:63,who:74,whole:81,whose:[54,81],why:74,wide:78,width:[74,82],window:74,window_nam:74,wish:75,within:[49,52,56,58,70,72,74],without:[60,61,72,74,83],wl:84,wooden:74,word:[74,82],work:[44,54,58,60,74,75,81,83],worker:83,workflow:[81,82,86],workspac:[49,52,63,70],workspace_s:[45,49,52,70],world:74,would:[52,60,61,62,63,81,84,85,86],wp:85,wrap:[56,57,58,61,74,77,81,86],wrapper:[60,81],write:[3,4,29,30,44,53,61,64,74,81,83,85],write_calibration_cach:68,writecalibrationcach:[3,4,44],wrote:74,www:[61,63,72,74,83,85],x86:84,x86_64:[58,63],x9:54,x:[5,10,33,43,54,61,63,70,75,80],x_0:74,x_1:74,x_2:74,x_3:74,x_4:74,x_:74,xavier:[45,87],xstr:[19,43,50],xx:85,xxx:81,y:[33,70,75],yahoo:75,yaml:59,yet:[81,82],you:[0,1,2,29,30,46,48,49,52,53,54,55,57,58,60,61,62,63,64,69,70,72,74,75,76,80,81,82,83,84,85,86],your:[60,61,62,63,64,72,74,75,79,80,84,86],yourself:61,yy:85,z:75,zero_point:65,zip:[57,63],zisserman:83},titles:["Class DataType","Class Device::DeviceType","Class TensorFormat","Template Class Int8CacheCalibrator","Template Class Int8Calibrator","Define STR","Define TORCH_TENSORRT_PATCH_VERSION","Define TORCH_TENSORRT_MAJOR_VERSION","Define TORCH_TENSORRT_MINOR_VERSION","Define TORCHTRT_API","Define XSTR","Define TORCHTRT_HIDDEN","Define TORCH_TENSORRT_VERSION","Directory cpp","Directory include","Directory torch_tensorrt","Enum Level","Enum EngineCapability","File logging.h","File macros.h","File ptq.h","File torch_tensorrt.h","Function torch_tensorrt::logging::get_logging_prefix","Function torch_tensorrt::logging::get_reportable_log_level","Function torch_tensorrt::logging::get_is_colored_output_on","Function torch_tensorrt::logging::set_reportable_log_level","Function torch_tensorrt::logging::log","Function torch_tensorrt::logging::set_is_colored_output_on","Function torch_tensorrt::logging::set_logging_prefix","Template Function torch_tensorrt::ptq::make_int8_cache_calibrator","Template Function torch_tensorrt::ptq::make_int8_calibrator","Function torch_tensorrt::torchscript::check_method_operator_support","Function torch_tensorrt::torchscript::compile","Function torch_tensorrt::torchscript::embed_engine_in_new_module","Function torch_tensorrt::get_build_info","Function torch_tensorrt::set_device","Function torch_tensorrt::dump_build_info","Function torch_tensorrt::torchscript::convert_method_to_trt_engine","Namespace torch_tensorrt","Namespace torch_tensorrt::logging","Namespace torch_tensorrt::ptq","Namespace torch_tensorrt::torchscript","Program Listing for File logging.h","Program Listing for File macros.h","Program Listing for File ptq.h","Program Listing for File torch_tensorrt.h","Struct Device","Struct GraphInputs","Struct Input","Struct CompileSpec","Torch-TensorRT C++ API","Full API","torchtrtc","Conversion Phase","Lowering Phase","Partitioning Phase","Compiler Phases","Runtime Phase","System Overview","Useful Links for Torch-TensorRT Development","Writing Converters","Using Torch-TensorRT in C++","Using Torch-TensorRT in Python","Installation","Torch-TensorRT","Operators Supported","torch_tensorrt.fx","torch_tensorrt.logging","torch_tensorrt.ptq","torch_tensorrt","torch_tensorrt.ts","Changelog","Configuration","5. :mod:`test_py_module`","3. Paragraph Level Markup","4. Lists & Tables","1. Long Sticky Nav","1. Structural Elements","<no title>","Installation","Creating a TorchScript Module","Torch-TensorRT (FX Frontend) User Guide","Example notebooks","Post Training Quantization (PTQ)","Deploying Torch-TensorRT Programs","Serving a Torch-TensorRT model with Triton","Using Torch-TensorRT Directly From PyTorch","DLA"],titleterms:{"1":[76,85],"10":76,"11":76,"12":76,"13":76,"14":76,"15":76,"16":76,"17":76,"18":76,"19":76,"2":[76,77,85],"20":76,"3":[76,85],"4":76,"5":76,"6":76,"7":76,"8":76,"9":76,"class":[0,1,2,3,4,20,21,38,40,41,50,66,68,69],"enum":[16,17,18,21,38,39,50,68,69],"function":[18,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,50,59,66,69,70],"long":[76,78],A:74,And:74,But:75,By:[18,19],Or:54,The:[61,74],To:54,aarch64:63,abi:[57,63],acc:81,acceler:82,add:81,addmm:54,admonit:74,advic:60,ahead:64,an:78,api:[50,51,59,63,64],applic:83,arg:[60,73],automat:55,avail:59,awar:82,background:[57,60],base:[3,4,48,72],bert:82,binari:63,block:74,branch:54,build:[63,72,85],bullet:75,c:[50,59,61,63,64,82,83],can:75,caption:[75,78],center:74,ch:74,changelog:71,check_method_operator_support:31,choos:63,citat:[74,83],citrinet:82,cli:[63,64],client:85,cmake:63,code:[54,74],compil:[32,56,58,61,63,64,82],compilespec:49,compound:74,configur:72,construct:57,content:[18,19,20,21,38,39,40,41,72,73,74,75,76,77],context:[60,72],contigu:54,contract:60,contributor:64,convers:[53,56,58,60],convert:[53,60,61,65,81],convert_method_to_trt_engin:37,cpp:[13,18,19,20,21,55],creat:[80,83],creativ:74,cudnn:63,current:65,custom:61,cxx11:63,data:73,datatyp:0,dead:54,debug:63,deep:82,deeper:75,defin:[5,6,7,8,9,10,11,12,19,50],definit:[18,19,20,21,75],demo:78,depend:63,deploi:[82,84],deseri:57,detect:82,develop:59,devic:[1,46],devicetyp:1,dimens:59,direct:74,directli:86,directori:[13,14,15,51],disk:80,distribut:63,dla:87,doctest:74,documen:64,document:[0,1,2,3,4,5,6,7,8,9,10,11,12,16,17,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,46,47,48,49,59,64,77,78],down:75,download:[74,79],dropout:54,dump_build_info:36,dynam:82,easier:59,efficentnet:82,element:77,elimin:54,eliminatecommonsubexpress:54,embed_engine_in_new_modul:33,emphas:74,engin:[57,81],enginecap:17,enumer:75,envior:63,evalu:[53,65],exampl:[74,76,82],execept:54,executor:57,expect:59,face:82,fallback:[54,55],field:75,figur:74,file:[15,18,19,20,21,42,43,44,45,50,51],flatten:54,footnot:74,format:57,freez:54,from:[63,86],frontend:[81,82],full:[50,51],fuse:54,fx2trt:81,fx:[66,81,82],gaurd:54,gener:73,get:64,get_build_info:34,get_is_colored_output_on:24,get_logging_prefix:22,get_reportable_log_level:23,giant:75,git:79,glossari:74,gpu:64,graph:[54,57],graphinput:47,grid:75,guarante:60,guid:81,h:[18,19,20,21,42,43,44,45,55],have:75,hierarchi:50,hlist:75,hole:75,hood:61,how:[72,81,83],html:72,hug:82,ien:74,imag:[74,75],includ:[14,18,19,20,21],incred:78,index:73,indic:64,infer:85,inherit:[3,4,48],inlin:74,input:48,instal:[63,79],int8:82,int8cachecalibr:3,int8calibr:4,ir:59,jetson:63,jit:64,languag:82,layer:59,learn:82,lenet:82,level:[16,72,74,75],librari:[63,84],libtorchtrt:84,like:75,line:74,linear:54,link:[59,74],list:[42,43,44,45,75],liter:74,local:63,log:[18,22,23,24,25,26,27,28,39,42,67],logsoftmax:54,loop:54,lower:[54,56,58],macro:[19,43],make_int8_cache_calibr:29,make_int8_calibr:30,markup:74,mask:82,math:74,menu:[76,78],meta:74,miss:81,mlm:82,mod:73,model:[81,82,85],modul:[54,61,80],namespac:[18,19,20,21,38,39,40,41,50],nativ:63,native_op:59,nav:76,nest:[1,46],node:53,notebook:82,number:[74,75],nvidia:64,object:82,one:75,op:[57,81],oper:[61,65],optim:85,optimz:54,option:[72,73,75],other:60,overview:58,own:83,packag:[63,84],page:72,paragraph:[74,77],paramet:73,partit:[55,56,58],partitoninfo:55,pass:54,pattern:54,peephol:54,phase:[53,54,55,56,57,58],plugin:84,post:83,pre:63,precompil:63,prerequisit:63,program:[42,43,44,45,84],project:72,ptq:[20,29,30,40,44,68,83],python:[59,62,63,64,80,83],pytorch:[59,64,81,82,86],quantiz:[82,83],queri:85,quickstart:61,quot:74,rabbit:75,read:59,redund:54,refer:74,regist:61,relationship:[1,3,4,46,48],releas:63,remov:54,replac:74,resnet50:82,respons:60,result:57,right:63,rubric:74,runtim:[56,57,58,84],save:80,second:75,section:77,segmentedblock:55,serial:57,serv:[82,85],server:85,set:85,set_devic:35,set_is_colored_output_on:27,set_logging_prefix:28,set_reportable_log_level:25,setup:63,shape:82,shape_analysi:55,sidebar:74,so:84,sometim:59,sourc:63,ssd:82,start:64,step:85,sticki:76,str:5,struct:[46,47,48,49,50],structur:77,subdirectori:[13,14],submenu:76,submodul:69,subsect:77,subsubmenu:76,subsubsect:77,support:65,system:58,tabl:[72,73,74,75,76,77],tarbal:63,target:74,templat:[3,4,29,30],tensorformat:2,tensorrt:[50,57,59,61,62,63,64,81,82,84,85,86],test_py_modul:73,text:74,theme:[72,78],thi:[75,78],through:65,time:64,titl:74,toc:72,topic:74,torch:[50,59,61,62,64,81,82,84,85,86],torch_tensorrt:[15,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,45,66,67,68,69,70],torch_tensorrt_major_vers:7,torch_tensorrt_minor_vers:8,torch_tensorrt_patch_vers:6,torch_tensorrt_vers:12,torchscript:[31,32,33,37,41,61,64,80],torchtrt_api:9,torchtrt_hidden:11,torchtrtc:[52,61],tracer:81,train:[82,83],transform:82,triton:85,ts:70,tupl:54,tutori:64,type:[3,4,46,48],under:61,unpack:54,unrol:54,unsupport:61,up:85,us:[54,59,61,62,63,82,86],user:81,version:57,via:79,wai:74,weight:60,what:60,wide:72,work:[61,80],write:60,xstr:10,your:[83,85]}}) \ No newline at end of file diff --git a/docs/src/pytorch-sphinx-theme/docs/changelog.html b/docs/src/pytorch-sphinx-theme/docs/changelog.html index 8dd17b4aaf..043176e7bc 100644 --- a/docs/src/pytorch-sphinx-theme/docs/changelog.html +++ b/docs/src/pytorch-sphinx-theme/docs/changelog.html @@ -9,22 +9,22 @@ - + Changelog — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - + @@ -395,12 +393,12 @@

    Changelog @@ -409,9 +407,9 @@

    Changelog + - - + @@ -422,7 +420,7 @@

    Changelog - + Configuration — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - + @@ -498,12 +496,12 @@

    How the Table of Contents builds @@ -512,9 +510,9 @@

    How the Table of Contents builds + - - + @@ -525,7 +523,7 @@

    How the Table of Contents builds - + 5. :mod:`test_py_module` — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • 5. :mod:`test_py_module`
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +
    @@ -449,12 +447,12 @@

    5.3.

    + + - - - + @@ -463,9 +461,9 @@

    5.3. + - - + @@ -476,7 +474,7 @@

    5.3. jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/demo.html b/docs/src/pytorch-sphinx-theme/docs/demo/demo.html index c451d5b566..107aed2ba9 100644 --- a/docs/src/pytorch-sphinx-theme/docs/demo/demo.html +++ b/docs/src/pytorch-sphinx-theme/docs/demo/demo.html @@ -11,22 +11,22 @@ - + 3. Paragraph Level Markup — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -36,11 +36,11 @@ - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • 3. Paragraph Level Markup
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    3. Paragraph Level Markup

    @@ -546,7 +544,7 @@

    3.4.4.

    3.4.5. Code Blocks

    # parsed-literal test
-curl -O http://someurl/release-master (1.2.0a0+ffedb78).tar-gz
    +curl -O http://someurl/release-master (1.2.0a0+0baf50a).tar-gz
    Code Blocks can have captions.
    {
@@ -919,16 +917,16 @@ 
    3.8. 
-
+
    @@ -1003,12 +1001,12 @@

    3.8.

    + + - - - + @@ -1019,9 +1017,9 @@

    3.8. + - - + @@ -1032,7 +1030,7 @@

    3.8. + diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html b/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html index c00055cc53..569f25a708 100644 --- a/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html +++ b/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html @@ -9,22 +9,22 @@ - + 4. Lists & Tables — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • 4. Lists & Tables
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    + + - - - + @@ -994,9 +992,9 @@

    4.2.2. + - - + @@ -1007,7 +1005,7 @@

    4.2.2. + diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/long.html b/docs/src/pytorch-sphinx-theme/docs/demo/long.html index 94b812b397..8daf5bfeab 100644 --- a/docs/src/pytorch-sphinx-theme/docs/demo/long.html +++ b/docs/src/pytorch-sphinx-theme/docs/demo/long.html @@ -9,22 +9,22 @@ - + 1. Long Sticky Nav — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • 1. Long Sticky Nav
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    + + - - - + @@ -673,9 +671,9 @@

    1.22.5. + - - + @@ -686,7 +684,7 @@

    1.22.5. + diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/structure.html b/docs/src/pytorch-sphinx-theme/docs/demo/structure.html index fb830d4fb1..a530aefca2 100644 --- a/docs/src/pytorch-sphinx-theme/docs/demo/structure.html +++ b/docs/src/pytorch-sphinx-theme/docs/demo/structure.html @@ -9,22 +9,22 @@ - + 1. Structural Elements — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
      - +
    • - + Docs - + >
      • - +
    • 1. Structural Elements
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    + + - - - + @@ -542,9 +540,9 @@

    2.1.1. + - - + @@ -555,7 +553,7 @@

    2.1.1. + diff --git a/docs/src/pytorch-sphinx-theme/docs/index.html b/docs/src/pytorch-sphinx-theme/docs/index.html index e81e7c620b..76e47ffa77 100644 --- a/docs/src/pytorch-sphinx-theme/docs/index.html +++ b/docs/src/pytorch-sphinx-theme/docs/index.html @@ -9,22 +9,22 @@ - + <no title> — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - +

    Theme Documentation

      @@ -445,16 +443,16 @@
    - +
    @@ -485,12 +483,12 @@
    + + - - - + @@ -499,9 +497,9 @@ + - - + @@ -512,7 +510,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/src/pytorch-sphinx-theme/docs/installing.html b/docs/src/pytorch-sphinx-theme/docs/installing.html index b9e6f1b0c6..5f25809215 100644 --- a/docs/src/pytorch-sphinx-theme/docs/installing.html +++ b/docs/src/pytorch-sphinx-theme/docs/installing.html @@ -9,22 +9,22 @@ - + Installation — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -34,11 +34,11 @@ - + + - - + @@ -178,9 +178,9 @@ + - - +
    Table of Contents @@ -191,21 +191,21 @@
    @@ -286,7 +284,7 @@
    - + @@ -305,30 +303,30 @@
    - +
    @@ -340,13 +338,13 @@
    - - + +
    - +
    - + @@ -407,12 +405,12 @@

    Via Git or Download @@ -421,9 +419,9 @@

    Via Git or Download + - - + @@ -434,7 +432,7 @@

    Via Git or Download - + Creating a TorchScript Module — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Creating a TorchScript Module
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    Creating a TorchScript Module

    TorchScript is a way to create serializable and optimizable models from PyTorch code. @@ -360,41 +358,44 @@ 

     1import torch.nn as nn
  2import torch.nn.functional as F
  3
- 4class LeNetFeatExtractor(nn.Module):
- 5    def __init__(self):
- 6        super(LeNetFeatExtractor, self).__init__()
- 7        self.conv1 = nn.Conv2d(1, 6, 3)
- 8        self.conv2 = nn.Conv2d(6, 16, 3)
- 9
-10    def forward(self, x):
-11        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
-12        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
-13        return x
-14
-15class LeNetClassifier(nn.Module):
-16    def __init__(self):
-17        super(LeNetClassifier, self).__init__()
-18        self.fc1 = nn.Linear(16 * 6 * 6, 120)
-19        self.fc2 = nn.Linear(120, 84)
-20        self.fc3 = nn.Linear(84, 10)
-21
-22    def forward(self, x):
-23        x = torch.flatten(x,1)
-24        x = F.relu(self.fc1(x))
-25        x = F.relu(self.fc2(x))
-26        x = self.fc3(x)
-27        return x
-28
-29class LeNet(nn.Module):
-30    def __init__(self):
-31        super(LeNet, self).__init__()
-32        self.feat = LeNetFeatExtractor()
-33        self.classifer = LeNetClassifier()
-34
-35    def forward(self, x):
-36        x = self.feat(x)
-37        x = self.classifer(x)
-38        return x
+ 4
+ 5class LeNetFeatExtractor(nn.Module):
+ 6    def __init__(self):
+ 7        super(LeNetFeatExtractor, self).__init__()
+ 8        self.conv1 = nn.Conv2d(1, 6, 3)
+ 9        self.conv2 = nn.Conv2d(6, 16, 3)
+10
+11    def forward(self, x):
+12        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
+13        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
+14        return x
+15
+16
+17class LeNetClassifier(nn.Module):
+18    def __init__(self):
+19        super(LeNetClassifier, self).__init__()
+20        self.fc1 = nn.Linear(16 * 6 * 6, 120)
+21        self.fc2 = nn.Linear(120, 84)
+22        self.fc3 = nn.Linear(84, 10)
+23
+24    def forward(self, x):
+25        x = torch.flatten(x, 1)
+26        x = F.relu(self.fc1(x))
+27        x = F.relu(self.fc2(x))
+28        x = self.fc3(x)
+29        return x
+30
+31
+32class LeNet(nn.Module):
+33    def __init__(self):
+34        super(LeNet, self).__init__()
+35        self.feat = LeNetFeatExtractor()
+36        self.classifer = LeNetClassifier()
+37
+38    def forward(self, x):
+39        x = self.feat(x)
+40        x = self.classifer(x)
+41        return x

    .

    @@ -407,7 +408,7 @@ 
    import torch
 
 model = LeNet()
-input_data = torch.empty([1,1,32,32])
+input_data = torch.empty([1, 1, 32, 32])
 traced_model = torch.jit.trace(model, input_data)
    @@ -467,25 +468,25 @@

    Saving TorchScript Module to Disk - - - - - - + + + + + +

    + - - +
    - +

    @@ -493,11 +494,11 @@

    Saving TorchScript Module to DiskSphinx using a theme provided by Read the Docs.

    - + @@ -519,12 +520,12 @@

    Saving TorchScript Module to Disk @@ -533,9 +534,9 @@

    Saving TorchScript Module to Disk + - - + @@ -546,7 +547,7 @@

    Saving TorchScript Module to Disk + + + + + + + + Torch-TensorRT (FX Frontend) User Guide — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + +
      + +
    • + + + Docs + + > +
      • + + +
    • Torch-TensorRT (FX Frontend) User Guide
      • + + +
    • + + + + + +
      • + +
    + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Torch-TensorRT (FX Frontend) User Guide

    +

    Torch-TensorRT (FX Frontend) is a tool that can convert a PyTorch model through torch.fx to an +TensorRT engine optimized targeting running on Nvidia GPUs. TensorRT is the inference engine +developed by NVIDIA which composed of various kinds of optimization including kernel fusion, +graph optimization, low precision, etc.. This tool is developed in Python environment which allows this +workflow to be very accessible to researchers and engineers. There are a few stages that a +user want to use this tool and we will introduce them here.

    +

    > Torch-TensorRT (FX Frontend) is in Beta and currently it is recommended to work with PyTorch nightly.

    +
    # Test an example by
+$ python py/torch_tensorrt/fx/example/lower_example.py
+
    +
    +
    +

    Converting a PyTorch Model to TensorRT Engine

    +

    In general, users are welcome to use the compile() to finish the conversion from a model to tensorRT engine. It is a +wrapper API that consists of the major steps needed to finish this converison. Please refer to lower_example.py file in examples/fx.

    +

    In this section, we will go through an example to illustrate the major steps that fx path uses. +Users can refer to fx2trt_example.py file in examples/fx.

    +
      +

    • Step 1: Trace the model with acc_tracer

      • +
    +

    Acc_tracer is a tracer inheritated from FX tracer. It comes with args normalizer to convert all args to kwargs and pass to TRT converters.

    +
    import torch_tensorrt.fx.tracer.acc_tracer.acc_tracer as acc_tracer
+
+# Build the model which needs to be a PyTorch nn.Module.
+my_pytorch_model = build_model()
+
+# Prepare inputs to the model. Inputs have to be a List of Tensors
+inputs = [Tensor, Tensor, ...]
+
+# Trace the model with acc_tracer.
+acc_mod = acc_tracer.trace(my_pytorch_model, inputs)
+
    +
    +

    Common Errors:

    +

    symbolically traced variables cannot be used as inputs to control flow +This means the model contains dynamic control flow. Please refer to section “Dynamic Control Flow” in FX guide.

    +
      +

    • Step 2: Build TensorRT engine

      • +
    +

    There are two different modes for how TensorRT handles batch dimension, explicit batch dimension and implicit batch dimension. This mode was used by early versions of TensorRT, and is now deprecated but continues to be supported for backwards compatibility. In explicit batch mode, all dimensions are explicit and can be dynamic, that is their length can change at execution time. Many new features, such as dynamic shapes and loops, are available only in this mode. User can still choose to use implicit batch mode when they set explicit_batch_dimension=False in compile(). We do not recommend to use it since it will lack of support in future TensorRT versions.

    +

    Explicit batch is the default mode and it must be set for dynamic shape. For most of vision task, user can choose to enable dynamic_batch in compile() if they want to get the similar effects as implicit mode where only batch dimension changes. It has some requirements: +1. Shapes of inputs, outputs and activations are fixed except batch dimension. +2. Inputs, outputs and activations have batch dimension as the major dimension. +3. All the operators in the model do not modify batch dimension (permute, transpose, split, etc.) or compute over batch dimension (sum, softmax, etc.).

    +

    For examples of the last path, if we have a 3D tensor t shaped as (batch, sequence, dimension), operations such as torch.transpose(0, 2). If any of these three are not satisfied, we’ll need to specify InputTensorSpec as inputs with dynamic range.

    +
    import deeplearning.trt.fx2trt.converter.converters
+from torch.fx.experimental.fx2trt.fx2trt import InputTensorSpec, TRTInterpreter
+
+# InputTensorSpec is a dataclass we use to store input information.
+# There're two ways we can build input_specs.
+# Option 1, build it manually.
+input_specs = [
+  InputTensorSpec(shape=(1, 2, 3), dtype=torch.float32),
+  InputTensorSpec(shape=(1, 4, 5), dtype=torch.float32),
+]
+# Option 2, build it using sample_inputs where user provide a sample
+inputs = [
+torch.rand((1,2,3), dtype=torch.float32),
+torch.rand((1,4,5), dtype=torch.float32),
+]
+input_specs = InputTensorSpec.from_tensors(inputs)
+
+# IMPORTANT: If dynamic shape is needed, we need to build it slightly differently.
+input_specs = [
+    InputTensorSpec(
+        shape=(-1, 2, 3),
+        dtype=torch.float32,
+        # Currently we only support one set of dynamic range. User may set other dimensions but it is not promised to work for any models
+        # (min_shape, optimize_target_shape, max_shape)
+        # For more information refer to fx/input_tensor_spec.py
+        shape_ranges = [
+            ((1, 2, 3), (4, 2, 3), (100, 2, 3)),
+        ],
+    ),
+    InputTensorSpec(shape=(1, 4, 5), dtype=torch.float32),
+]
+
+# Build a TRT interpreter. Set explicit_batch_dimension accordingly.
+interpreter = TRTInterpreter(
+    acc_mod, input_specs, explicit_batch_dimension=True/False
+)
+
+# The output of TRTInterpreter run() is wrapped as TRTInterpreterResult.
+# The TRTInterpreterResult contains required parameter to build TRTModule,
+# and other informational output from TRTInterpreter run.
+class TRTInterpreterResult(NamedTuple):
+    engine: Any
+    input_names: Sequence[str]
+    output_names: Sequence[str]
+    serialized_cache: bytearray
+
+#max_batch_size: set accordingly for maximum batch size you will use.
+#max_workspace_size: set to the maximum size we can afford for temporary buffer
+#lower_precision: the precision model layers are running on (TensorRT will choose the best perforamnce precision).
+#sparse_weights: allow the builder to examine weights and use optimized functions when weights have suitable sparsity
+#force_fp32_output: force output to be fp32
+#strict_type_constraints: Usually we should set it to False unless we want to control the precision of certain layer for numeric #reasons.
+#algorithm_selector: set up algorithm selection for certain layer
+#timing_cache: enable timing cache for TensorRT
+#profiling_verbosity: TensorRT logging level
+trt_interpreter_result = interpreter.run(
+    max_batch_size=64,
+    max_workspace_size=1 << 25,
+    sparse_weights=False,
+    force_fp32_output=False,
+    strict_type_constraints=False,
+    algorithm_selector=None,
+    timing_cache=None,
+    profiling_verbosity=None,
+)
+
    +
    +

    Common Errors:

    +

    RuntimeError: Conversion of function xxx not currently supported! +- This means we don’t have the support for this xxx operator. Please refer to section “How to add a missing op” below for further instructions.

    +
      +

    • Step 3: Run the model

      • +
    +

    One way is using TRTModule, which is basically a PyTorch nn.Module.

    +
    from torch_tensorrt.fx import TRTModule
+mod = TRTModule(
+    trt_interpreter_result.engine,
+    trt_interpreter_result.input_names,
+    trt_interpreter_result.output_names)
+# Just like all other PyTorch modules
+outputs = mod(*inputs)
+torch.save(mod, "trt.pt")
+reload_trt_mod = torch.load("trt.pt")
+reload_model_output = reload_trt_mod(*inputs)
+
    +
    +

    So far, we give a detailed explanation of major steps in convterting a PyTorch model into TensorRT engine. Users are welcome to refer to the source code for some parameters explanations. In the converting scheme, there are two important actions in it. One is acc tracer which helps us to convert a PyTorch model to acc graph. The other is FX path converter which helps to convert the acc graph’s operation to corresponding TensorRT operation and build up the TensoRT engine for it.

    +
    +
    +

    Acc Tracer

    +

    Acc tracer is a custom FX symbolic tracer. It does a couple more things compare to the vanilla FX symbolic tracer. We mainly depend on it to convert PyTorch ops or builtin ops to acc ops. There are two main purposes for fx2trt to use acc ops:

    +
      +

    1. there’re many ops that do similar things in PyTorch ops and builtin ops such like torch.add, builtin.add and torch.Tensor.add. Using acc tracer, we normalize these three ops to a single acc_ops.add. This helps reduce the number of converters we need to write.

      2. +

    2. acc ops only have kwargs which makes writing converter easier as we don’t need to add additional logic to find arguments in args and kwargs.

      3. +
    +
    +
    +

    FX2TRT

    +

    After symbolic tracing, we have the graph representation of a PyTorch model. fx2trt leverages the power of fx.Interpreter. fx.Interpreter goes through the whole graph node by node and calls the function that node represents. fx2trt overrides the original behavior of calling the function with invoking corresponding converts for each node. Each converter function adds corresponding TensorRT layer(s).

    +

    Below is an example of a converter function. The decorator is used to register this converter function with the corresponding node. In this example, we register this converter to a fx node whose target is acc_ops.sigmoid.

    +
    @tensorrt_converter(acc_ops.sigmoid)
+def acc_ops_sigmoid(network, target, args, kwargs, name):
+    """
+    network: TensorRT network. We'll be adding layers to it.
+
+    The rest arguments are attributes of fx node.
+    """
+    input_val = kwargs['input']
+
+    if not isinstance(input_val, trt.tensorrt.ITensor):
+        raise RuntimeError(f'Sigmoid received input {input_val} that is not part '
+                        'of the TensorRT region!')
+
+    layer = network.add_activation(input=input_val, type=trt.ActivationType.SIGMOID)
+    layer.name = name
+    return layer.get_output(0)
+
    +
    +
    +

    How to Add a Missing Op

    +

    You can actually add it wherever you want just need to remember import the file so that all acc ops and mapper will be registered before tracing with acc_tracer.

    +
      +

    • Step 1. Add a new acc op

      • +
    +

    TODO: Need to explain more on the logistic of acc op like when we want to break down an op and when we want to reuse other ops.

    +

    In acc tracer, we convert nodes in the graph to acc ops if there’s a mapping registered for the node to an acc op.

    +

    In order to make the conversion to acc ops to happen, there’re two things required. One is that there should be an acc op function defined and the other is there should be a mapping registered.

    +

    Defining an acc op is simple, we first just need a function and register the function as an acc op via this decorator acc_normalizer.py. e.g. the following code adds an acc op named foo() which adds two given inputs.

    +
    # NOTE: all acc ops should only take kwargs as inputs, therefore we need the "*"
+# at the beginning.
+@register_acc_op
+def foo(*, input, other, alpha):
+    return input + alpha * other
+
    +
    +

    There’re two ways to register a mapping. One is register_acc_op_mapping(). Let’s register a mapping from torch.add to foo() we just created above. We need to add decorator register_acc_op_mapping to it.

    +
    this_arg_is_optional = True
+
+@register_acc_op_mapping(
+    op_and_target=("call_function", torch.add),
+    arg_replacement_tuples=[
+        ("input", "input"),
+        ("other", "other"),
+        ("alpha", "alpha", this_arg_is_optional),
+    ],
+)
+@register_acc_op
+def foo(*, input, other, alpha=1.0):
+    return input + alpha * other
+
    +
    +

    op_and_target determines which node will trigger this mapping. op and target are the attributes of FX node. In acc_normalization when we see a node with the same op and target as set in the op_and_target, we will trigger the mapping. Since we want to map from torch.add, then op would be call_function and target would be torch.add. arg_replacement_tuples determines how we construct kwargs for new acc op node using args and kwargs from original node. Each tuple in arg_replacement_tuples represents one argument mapping rule. It contains two or three elements. The third element is a boolean variable that determines whether this kwarg is optional in original node. We only need to specify the third element if it’s True. The first element is the argument name in original node which will be used as the acc op node’s argument whose name is the second element in the tuple. The sequence of the tuples does matter because the position of the tuple determines where the argument is in original node’s args. We use this information to map args from original node to kwargs in acc op node. +We don’t have to specify arg_replacement_tuples if none of the followings are true.

    +
      +

    1. kwargs of original nodes and acc op nodes have different name.

      2. +

    2. there’re optional arguments.

      3. +
    +

    The other way to register a mapping is through register_custom_acc_mapper_fn(). This one is designed to reduce the redundant op registration as it allows you to use a function to map to one or more existing acc ops throught some combinations. In the function, you can do basically whatever you want. Let’s use an example to explain how it works.

    +
    @register_acc_op
+def foo(*, input, other, alpha=1.0):
+    return input + alpha * other
+
+@register_custom_acc_mapper_fn(
+    op_and_target=("call_function", torch.add),
+    arg_replacement_tuples=[
+        ("input", "input"),
+        ("other", "other"),
+        ("alpha", "alpha", this_arg_is_optional),
+    ],
+)
+def custom_mapper(node: torch.fx.Node, _: nn.Module) -> torch.fx.Node:
+    """
+    `node` is original node, which is a call_function node with target
+    being torch.add.
+    """
+    alpha = 1
+    if "alpha" in node.kwargs:
+        alpha = node.kwargs["alpha"]
+    foo_kwargs = {"input": node["input"], "other": node["other"], "alpha": alpha}
+    with node.graph.inserting_before(node):
+        foo_node = node.graph.call_function(foo, kwargs=foo_kwargs)
+        foo_node.meta = node.meta.copy()
+        return foo_node
+
    +
    +

    In the custom mapper function, we construct an acc op node and return it. The node we returns here would take over all the children nodes of original nodes acc_normalizer.py.

    +

    The last step would be adding unit test for the new acc op or mapper function we added. The place to add the unit test is here test_acc_tracer.py.

    +
      +

    • Step 2. Add a new converter

      • +
    +

    All the developed converters for acc ops are all in acc_op_converter.py. It could give you a good example of how the converter is added.

    +

    Essentially, the converter is the mapping mechanism that maps the acc ops to a TensorRT layer. If we are able to find all the TensorRT layers we need we can get start to add a converter for the node using TensorRT APIs.

    +
    @tensorrt_converter(acc_ops.sigmoid)
+def acc_ops_sigmoid(network, target, args, kwargs, name):
+    """
+    network: TensorRT network. We'll be adding layers to it.
+
+    The rest arguments are attributes of fx node.
+    """
+    input_val = kwargs['input']
+
+    if not isinstance(input_val, trt.tensorrt.ITensor):
+        raise RuntimeError(f'Sigmoid received input {input_val} that is not part '
+                        'of the TensorRT region!')
+
+    layer = network.add_activation(input=input_val, type=trt.ActivationType.SIGMOID)
+    layer.name = name
+    return layer.get_output(0)
+
    +
    +

    We need to use tensorrt_converter decorator to register the converter. The argument for the decorator is the target of the fx node that we need to convert. In the converter, we can find the inputs to the fx node in kwargs. As in the example, the original node is acc_ops.sigmoid which only has one argument “input” in acc_ops.py. We get the input and check if it’s a TensorRT tensor. After that, we add a sigmoid layer to TensorRT network and return the output of the layer. The output we returned will be passed to the children nodes of acc_ops.sigmoid by fx.Interpreter.

    +

    What if we can not find corresponding layers in TensorRT that do the same thing as the node.

    +

    In this case, we would need to do a bit more work. TensorRT provides plugins which serves as custom layers. We have not implement this feature yet. We will update once it is enabled.

    +

    Last step would be adding the unit test for the new converter we added. User could add corresponding unit test in this folder.

    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/tutorials/notebooks.html b/docs/tutorials/notebooks.html new file mode 100644 index 0000000000..12f550f445 --- /dev/null +++ b/docs/tutorials/notebooks.html @@ -0,0 +1,795 @@ + + + + + + + + + + + + + Example notebooks — Torch-TensorRT master documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Table of Contents + +
    + + + +
    +
    +
    + + + + + + + + + + + + + + + + +
    + +
      + +
    • + + + Docs + + > +
      • + + +
    • Example notebooks
      • + + +
    • + + + + + +
      • + +
    + + +
    +
    + +
    + Shortcuts +
    +
    + +
    +
    + + + +
    + +
    +
    + +
    +

    Example notebooks

    +

    There exists a number of notebooks which cover specific using specific features and models +with Torch-TensorRT

    +
    +

    Notebooks

    +
    +

    Compiling CitriNet with Torch-TensorRT

    +

    Citrinet is an acoustic model used for the speech to text recognition task. It is a version +of QuartzNet that extends ContextNet, utilizing subword encoding (via Word Piece tokenization) +and Squeeze-and-Excitation(SE) mechanism and are therefore smaller than QuartzNet models. CitriNet +models take in audio segments and transcribe them to letter, byte pair, or word piece sequences.

    +

    This notebook demonstrates the steps for optimizing a pretrained CitriNet model with Torch-TensorRT, +and running it to test the speedup obtained.

    + +
    +
    +

    Compiling EfficentNet with Torch-TensorRT

    +

    EfficentNet is a feedforward CNN designed to achieve better performance and accuracy than alternative architectures +by using a “scaling method that uniformly scales all dimensions of depth/width/resolution using a simple yet highly effective compound coefficient”.

    +

    This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT, +and running it to test the speedup obtained.

    + +
    +
    +

    Masked Language Modeling (MLM) with Hugging Face BERT Transformer accelerated by Torch-TensorRT

    +

    “BERT is a transformer model pretrained on a large corpus of English data in a self-supervised fashion. +This way, the model learns an inner representation of the English language that can then be used to extract +features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train +a standard classifier using the features produced by the BERT model as inputs.” (https://huggingface.co/bert-base-uncased)

    +

    This notebook demonstrates the steps for optimizing a pretrained EfficentNet model with Torch-TensorRT, +and running it to test the speedup obtained.

    + +
    +
    +

    Serving a model in C++ using Torch-TensorRT

    +

    This example shows how you can load a pretrained ResNet-50 model, convert it to a Torch-TensorRT +optimized model (via the Torch-TensorRT Python API), save the model as a torchscript module, and +then finally load and serve the model with the PyTorch C++ API.

    + +
    +
    +

    Compiling ResNet50 with Torch-TensorRT

    +

    This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a +pretrained ResNet-50 network, and running it to test the speedup obtained.

    + +
    +
    +

    Using Dynamic Shapes with Torch-TensorRT

    +

    Making use of Dynamic Shaped Tensors in Torch TensorRT is quite simple. Let’s say you are +using the torch_tensorrt.compile(...) function to compile a torchscript module. One +of the args in this function in this function is input: which defines an input to a +module in terms of expected shape, data type and tensor format: torch_tensorrt.Input.

    +

    For the purposes of this walkthrough we just need three kwargs: min_shape, opt_shape` and max_shape.

    +
    torch_tensorrt.Input(
+        min_shape=(1, 224, 224, 3),
+        opt_shape=(1, 512, 512, 3),
+        max_shape=(1, 1024, 1024, 3),
+        dtype=torch.int32
+        format=torch.channel_last
+    )
+...
+
    +
    +

    In this example, we are going to use a simple ResNet model to demonstrate the use of the API.

    + +
    +
    +

    Using the FX Frontend with Torch-TensorRT

    +

    The purpose of this example is to demostrate the overall flow of lowering a PyTorch model to TensorRT +conveniently with using FX.

    + +
    +
    +

    Compiling a PyTorch model using FX Frontend with Torch-TensorRT

    +

    The purpose of this example is to demonstrate the overall flow of lowering a PyTorch +model to TensorRT via FX with existing FX based tooling

    + +
    +
    +

    Compiling LeNet with Torch-TensorRT

    +

    This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a simple LeNet network.

    + +
    +
    +

    Accelerate Deep Learning Models using Quantization in Torch-TensorRT

    +

    Model Quantization is a popular way of optimization which reduces the size of models thereby +accelerating inference, also opening up the possibilities of deployments on devices with lower +computation power such as Jetson. Simply put, quantization is a process of mapping input values

    +
    +

    from a larger set to output values in a smaller set. In this notebook, we illustrate the workflow +that you can adopt while quantizing a deep learning model in Torch-TensorRT. The notebook takes +you through an example of Mobilenetv2 for a classification task on a subset of Imagenet Dataset +called Imagenette which has 10 classes.

    +
    + +
    +
    +

    Object Detection with Torch-TensorRT (SSD)

    +

    This notebook demonstrates the steps for compiling a TorchScript module with Torch-TensorRT on a pretrained SSD network, and running it to test the speedup obtained.

    + +
    +
    +

    Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT

    +

    Quantization Aware training (QAT) simulates quantization during training by +quantizing weights and activation layers. This will help to reduce the loss in +accuracy when we convert the network trained in FP32 to INT8 for faster inference. +QAT introduces additional nodes in the graph which will be used to learn the dynamic +ranges of weights and activation layers. In this notebook, we illustrate the following +steps from training to inference of a QAT model in Torch-TensorRT.

    + +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +

    Docs

    +

    Access comprehensive developer documentation for PyTorch

    + View Docs +
    + +
    +

    Tutorials

    +

    Get in-depth tutorials for beginners and advanced developers

    + View Tutorials +
    + +
    +

    Resources

    +

    Find development resources and get your questions answered

    + View Resources +
    +
    +
    +
    + + + +
    +
    +

    To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.

    + +
    +
    + + + + + +
    +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    + + + + + + + + \ No newline at end of file diff --git a/docs/tutorials/ptq.html b/docs/tutorials/ptq.html index 6ce21f0764..32c9f80d36 100644 --- a/docs/tutorials/ptq.html +++ b/docs/tutorials/ptq.html @@ -9,22 +9,22 @@ - + Post Training Quantization (PTQ) — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Post Training Quantization (PTQ)
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    Post Training Quantization (PTQ)

    Post Training Quantization (PTQ) is a technique to reduce the required computational resources for inference @@ -456,8 +454,6 @@ compile_spec.enabled_precisions.insert(torch::kI8); /// Use the TensorRT Entropy Calibrator compile_spec.ptq_calibrator = calibrator; -/// Set a larger workspace (you may get better performace from doing so) -compile_spec.workspace_size = 1 << 28; auto trt_mod = torch_tensorrt::CompileGraph(mod, compile_spec);

    @@ -471,51 +467,48 @@

    How to create your own PTQ application in Python

    Torch-TensorRT Python API provides an easy and convenient way to use pytorch dataloaders with TensorRT calibrators. DataLoaderCalibrator class can be used to create a TensorRT calibrator by providing desired configuration. The following code demonstrates an example on how to use it

    -
    testing_dataset = torchvision.datasets.CIFAR10(root='./data',
-                                                        train=False,
-                                                        download=True,
-                                                        transform=transforms.Compose([
-                                                            transforms.ToTensor(),
-                                                            transforms.Normalize((0.4914, 0.4822, 0.4465),
-                                                                                 (0.2023, 0.1994, 0.2010))
-                                                        ]))
-
-testing_dataloader = torch.utils.data.DataLoader(testing_dataset,
-                                                      batch_size=1,
-                                                      shuffle=False,
-                                                      num_workers=1)
-calibrator = torch_tensorrt.ptq.DataLoaderCalibrator(testing_dataloader,
-                                              cache_file='./calibration.cache',
-                                              use_cache=False,
-                                              algo_type=torch_tensorrt.ptq.CalibrationAlgo.ENTROPY_CALIBRATION_2,
-                                              device=torch.device('cuda:0'))
-
-compile_spec = {
-         "inputs": [torch_tensorrt.Input((1, 3, 32, 32))],
-         "enabled_precisions": {torch.float, torch.half, torch.int8},
-         "calibrator": calibrator,
-         "device": {
-             "device_type": torch_tensorrt.DeviceType.GPU,
-             "gpu_id": 0,
-             "dla_core": 0,
-             "allow_gpu_fallback": False,
-             "disable_tf32": False
-         }
-     }
-trt_mod = torch_tensorrt.compile(model, compile_spec)
+
    testing_dataset = torchvision.datasets.CIFAR10(
+    root="./data",
+    train=False,
+    download=True,
+    transform=transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
+        ]
+    ),
+)
+
+testing_dataloader = torch.utils.data.DataLoader(
+    testing_dataset, batch_size=1, shuffle=False, num_workers=1
+)
+calibrator = torch_tensorrt.ptq.DataLoaderCalibrator(
+    testing_dataloader,
+    cache_file="./calibration.cache",
+    use_cache=False,
+    algo_type=torch_tensorrt.ptq.CalibrationAlgo.ENTROPY_CALIBRATION_2,
+    device=torch.device("cuda:0"),
+)
+
+trt_mod = torch_tensorrt.compile(model, inputs=[torch_tensorrt.Input((1, 3, 32, 32))],
+                                    enabled_precisions={torch.float, torch.half, torch.int8},
+                                    calibrator=calibrator,
+                                    device={
+                                         "device_type": torch_tensorrt.DeviceType.GPU,
+                                         "gpu_id": 0,
+                                         "dla_core": 0,
+                                         "allow_gpu_fallback": False,
+                                         "disable_tf32": False
+                                     })
 
    
 
    
 
    In the cases where there is a pre-existing calibration cache file that users want to use, CacheCalibrator can be used without any dataloaders. The following example demonstrates how
 to use CacheCalibrator to use in INT8 mode.
    
 
    calibrator = torch_tensorrt.ptq.CacheCalibrator("./calibration.cache")
 
-compile_settings = {
-      "inputs": [torch_tensorrt.Input([1, 3, 32, 32])],
-      "enabled_precisions": {torch.float, torch.half, torch.int8},
-      "calibrator": calibrator,
-  }
-
-trt_mod = torch_tensorrt.compile(model, compile_settings)
+trt_mod = torch_tensorrt.compile(model, inputs=[torch_tensorrt.Input([1, 3, 32, 32])],
+                                      enabled_precisions={torch.float, torch.half, torch.int8},
+                                      calibrator=calibrator)
 
    
 
    
 
    If you already have an existing calibrator class (implemented directly using TensorRT API), you can directly set the calibrator field to your class which can be very convenient.
@@ -531,25 +524,25 @@ 
    Citations
-
-        
-
-
-        
-
+      
+        
+      
+      
+        
+
    + - - +
    - +

    @@ -557,11 +550,11 @@

    CitationsSphinx using a theme provided by Read the Docs.

    - + @@ -588,12 +581,12 @@

    Citations @@ -602,9 +595,9 @@

    Citations + - - + @@ -615,7 +608,7 @@

    Citations - + Deploying Torch-TensorRT Programs — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Deploying Torch-TensorRT Programs
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    Deploying Torch-TensorRT Programs

    After compiling and saving Torch-TensorRT programs there is no longer a strict dependency on the full @@ -368,7 +366,7 @@

    Runtime Library

    Note

    -

    If you are using the standard distribution of PyTorch in Python on x86, likely you will need the pre-cxx11-abi variant of libtorchtrt_runtime.so, check Installation documentation for more details.

    +

    If you are using the standard distribution of PyTorch in Python on x86, likely you will need the pre-cxx11-abi variant of libtorchtrt_runtime.so, check Installation documentation for more details.

    Note

    @@ -386,25 +384,25 @@

    Plugin Library - - - - - - + + + + + +

    + - - +
    - +

    @@ -412,11 +410,11 @@

    Plugin LibrarySphinx using a theme provided by Read the Docs.

    - + @@ -441,12 +439,12 @@

    Plugin Library @@ -455,9 +453,9 @@

    Plugin Library + - - + @@ -468,7 +466,7 @@

    Plugin Library - + Serving a Torch-TensorRT model with Triton — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Serving a Torch-TensorRT model with Triton
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    Serving a Torch-TensorRT model with Triton

    Optimization and deployment go hand in hand in a discussion about Machine @@ -533,25 +531,25 @@

    Step 3: Building a Triton Client to Query the Server - +

    @@ -588,12 +586,12 @@

    Step 3: Building a Triton Client to Query the Server

    + + - - - + @@ -602,9 +600,9 @@

    Step 3: Building a Triton Client to Query the Server + - - + @@ -615,7 +613,7 @@

    Step 3: Building a Triton Client to Query the Server + diff --git a/docs/tutorials/use_from_pytorch.html b/docs/tutorials/use_from_pytorch.html index af73b70ac4..10f7966f4e 100644 --- a/docs/tutorials/use_from_pytorch.html +++ b/docs/tutorials/use_from_pytorch.html @@ -9,22 +9,22 @@ - + Using Torch-TensorRT Directly From PyTorch — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
      - +
    • - + Docs - + >
      • - +
    • Using Torch-TensorRT Directly From PyTorch
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -342,17 +340,17 @@
    - - + +
    - +
    - +

    Using Torch-TensorRT Directly From PyTorch

    You will now be able to directly access TensorRT from PyTorch APIs. The process to use this feature -is very similar to the compilation workflow described in Using Torch-TensorRT in Python

    +is very similar to the compilation workflow described in Using Torch-TensorRT in Python

    Start by loading torch_tensorrt into your application.

    import torch
 import torch_tensorrt
@@ -371,8 +369,8 @@
 sort of dictionary you would provide to compile. For more information on the compile spec dictionary take a look
 at the documentation for the Torch-TensorRT TensorRTCompileSpec API.
    
 
    spec = {
-    "forward":
-        torch_tensorrt.ts.TensorRTCompileSpec({
+    "forward": torch_tensorrt.ts.TensorRTCompileSpec(
+        {
             "inputs": [torch_tensorrt.Input([1, 3, 300, 300])],
             "enabled_precisions": {torch.float, torch.half},
             "refit": False,
@@ -381,13 +379,13 @@
                 "device_type": torch_tensorrt.DeviceType.GPU,
                 "gpu_id": 0,
                 "dla_core": 0,
-                "allow_gpu_fallback": True
+                "allow_gpu_fallback": True,
             },
             "capability": torch_tensorrt.EngineCapability.default,
-            "num_min_timing_iters": 2,
             "num_avg_timing_iters": 1,
-        })
-    }
+        }
+    )
+}
 
    
 
    
 
    Now to compile with Torch-TensorRT, provide the target module objects and the spec dictionary to torch._C._jit_to_backend("tensorrt", ...)
    
@@ -403,25 +401,25 @@
    - +
    @@ -453,12 +451,12 @@
    + + - - - + @@ -467,9 +465,9 @@ + - - + @@ -480,7 +478,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + diff --git a/docs/tutorials/using_dla.html b/docs/tutorials/using_dla.html index 4915aa1b66..c1e89c8386 100644 --- a/docs/tutorials/using_dla.html +++ b/docs/tutorials/using_dla.html @@ -9,22 +9,22 @@ - + DLA — Torch-TensorRT master documentation + + + + + + + + + - - - - - - - - - - + @@ -33,14 +33,14 @@ - - + + - + + - - + @@ -180,9 +180,9 @@ + - - +
    Table of Contents @@ -193,21 +193,21 @@
    @@ -288,7 +286,7 @@
    - + @@ -307,30 +305,30 @@
    - +
    @@ -342,13 +340,13 @@
    - - + +
    - +
    - +

    DLA

    DLA NVIDIA Deep Learning Accelerator is a fixed-function accelerator engine targeted for deep learning operations. DLA is designed to do full hardware acceleration of convolutional neural networks. DLA supports various layers such as convolution, deconvolution, fully-connected, activation, pooling, batch normalization, etc. torch_tensorrt supports compilation of TorchScript Module and deployment pipeline on the DLA hardware available on NVIDIA embedded platforms.

    @@ -373,16 +371,13 @@ # If a layer fails to run on DLA it will fallback to GPU compile_spec.device.allow_gpu_fallback = true; - -# Set the workspace size -compile_spec.workspace_size = 1 << 28;

    Using DLA in a python application

    compile_spec = {
     "inputs": [torch_tensorrt.Input(self.input.shape)],
     "device": torch_tensorrt.Device("dla:0", allow_gpu_fallback=True),
-    "enalbed_precisions": {torch.half}
+    "enalbed_precisions": {torch.half},
 }
 
 trt_mod = torch_tensorrt.compile(self.scripted_model, compile_spec)
@@ -392,25 +387,25 @@

    - +
    @@ -442,12 +437,12 @@
    + + - - - + @@ -456,9 +451,9 @@ + - - + @@ -469,7 +464,7 @@ jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); - + From 524876828caf946fc3452afe3db0081b3f2a56e8 Mon Sep 17 00:00:00 2001 From: Naren Dasan <1790613+narendasan@users.noreply.github.com> Date: Mon, 22 Aug 2022 18:30:30 -0700 Subject: [PATCH 31/31] fix(tools): Fix linter to not depend on docker (#1301) (#1302) * fix(tools): Fix linter to not depend on docker Signed-off-by: Naren Dasan <1790613+narendasan@users.noreply.github.com> * Update linter.yml * Update linter.yml Signed-off-by: Naren Dasan <1790613+narendasan@users.noreply.github.com> Signed-off-by: Naren Dasan <1790613+narendasan@users.noreply.github.com> --- .github/workflows/linter.yml | 44 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index da05d2f41a..58c8440684 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -25,25 +25,25 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Docker login - uses: docker/login-action@v1 + - name: Set up Python 3.9.4 + uses: actions/setup-python@v2 with: - registry: docker.pkg.github.com - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Run image + python-version: 3.9.4 + - name: Setup env run: | - docker pull docker.pkg.github.com/pytorch/tensorrt/docgen:latest - docker run -it -d --name cpplinter -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GITHUB_WORKSPACE:/workspace -v $GITHUB_EVENT_PATH:/GITHUB_EVENT.json -w /workspace docker.pkg.github.com/pytorch/tensorrt/docgen:latest - docker exec cpplinter bash -c "cp /workspace/docker/WORKSPACE.docker /workspace/WORKSPACE" + sudo ln -s $GITHUB_EVENT_PATH /GITHUB_EVENT.json env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Install dependencies run: | - docker exec cpplinter bash -c "pip3 install -r /workspace/.github/scripts/requirements.txt" - docker exec cpplinter bash -c "pip3 install -r /workspace/requirements-dev.txt" + pip3 install -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt + pip3 install -r $GITHUB_WORKSPACE/requirements-dev.txt - name: Lint C++ - run: docker exec cpplinter bash -c "cd /workspace && python3 /workspace/.github/scripts/run_cpp_linter.py" + run: | + cd $GITHUB_WORKSPACE + python3 $GITHUB_WORKSPACE/.github/scripts/run_cpp_linter.py + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} py-linting: name: Python Linting @@ -65,22 +65,28 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python 3.9.4 + uses: actions/setup-python@v2 + with: + python-version: 3.9.4 - name: Docker login uses: docker/login-action@v1 with: registry: docker.pkg.github.com username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Run image + - name: Setup env run: | - docker pull docker.pkg.github.com/pytorch/tensorrt/docgen:latest - docker run -it -d --name pylinter -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GITHUB_WORKSPACE:/workspace -v $GITHUB_EVENT_PATH:/GITHUB_EVENT.json -w /workspace docker.pkg.github.com/pytorch/tensorrt/docgen:latest - docker exec pylinter bash -c "cp /workspace/docker/WORKSPACE.docker /workspace/WORKSPACE" + sudo ln -s $GITHUB_EVENT_PATH /GITHUB_EVENT.json env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Install dependencies run: | - docker exec pylinter bash -c "pip3 install -r /workspace/.github/scripts/requirements.txt" - docker exec pylinter bash -c "pip3 install -r /workspace/requirements-dev.txt" + pip3 install -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt + pip3 install -r $GITHUB_WORKSPACE/requirements-dev.txt - name: Lint Python - run: docker exec pylinter bash -c "cd /workspace && python3 /workspace/.github/scripts/run_py_linter.py" + run: | + cd $GITHUB_WORKSPACE + python3 $GITHUB_WORKSPACE/.github/scripts/run_py_linter.py + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}