Merge remote-tracking branch 'upstream/master' into IFU-master-2022-04-26

Author: root

.circleci/cimodel/data/pytorch_build_data.py

@@ -74,7 +74,7 @@ def child_constructor(self):
             "mlc": MLCConfigNode,
             "vulkan": VulkanConfigNode,
             "parallel_tbb": ParallelTBBConfigNode,
-            "noarch": NoarchConfigNode,
+            "crossref": CrossRefConfigNode,
             "parallel_native": ParallelNativeConfigNode,
             "onnx": ONNXConfigNode,
             "libtorch": LibTorchConfigNode,
@@ -171,9 +171,9 @@ def child_constructor(self):
         return ImportantConfigNode
 
 
-class NoarchConfigNode(TreeConfigNode):
+class CrossRefConfigNode(TreeConfigNode):
     def init2(self, node_name):
-        self.props["is_noarch"] = node_name
+        self.props["is_crossref"] = node_name
 
     def child_constructor(self):
         return ImportantConfigNode

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -239,7 +239,7 @@ def instantiate_configs(only_slow_gradcheck):
         compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
         is_asan = fc.find_prop("is_asan") or False
-        is_noarch = fc.find_prop("is_noarch") or False
+        is_crossref = fc.find_prop("is_crossref") or False
         is_onnx = fc.find_prop("is_onnx") or False
         is_pure_torch = fc.find_prop("is_pure_torch") or False
         is_vulkan = fc.find_prop("is_vulkan") or False
@@ -283,8 +283,8 @@ def instantiate_configs(only_slow_gradcheck):
             python_version = fc.find_prop("pyver")
             parms_list[0] = fc.find_prop("abbreviated_pyver")
 
-        if is_noarch:
-            parms_list_ignored_for_docker_image.append("noarch")
+        if is_crossref:
+            parms_list_ignored_for_docker_image.append("crossref")
 
         if is_onnx:
             parms_list.append("onnx")

.circleci/docker/build.sh

@@ -134,6 +134,18 @@ case "$image" in
     VISION=yes
     KATEX=yes
     ;;
+  pytorch-linux-bionic-cuda11.3-cudnn8-py3-clang9)
+    CUDA_VERSION=11.3.0 # Deviating from major.minor to conform to nvidia's Docker image names
+    CUDNN_VERSION=8
+    TENSORRT_VERSION=8.0.1.6
+    ANACONDA_PYTHON_VERSION=3.7
+    CMAKE_VERSION=3.10.3
+    CLANG_VERSION=9
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    KATEX=yes
+    ;;
   pytorch-linux-bionic-cuda11.5-cudnn8-py3-gcc7)
     CUDA_VERSION=11.5.0
     CUDNN_VERSION=8

.circleci/scripts/cpp_doc_push_script.sh

@@ -20,7 +20,7 @@ echo "cpp_doc_push_script.sh: Invoked with $*"
 #       but since DOCS_INSTALL_PATH can be derived from DOCS_VERSION it's probably better to
 #       try and gather it first, just so we don't potentially break people who rely on this script
 # Argument 2: What version of the Python API docs we are building.
-version="${2:-${DOCS_VERSION:-main}}"
+version="${2:-${DOCS_VERSION:-master}}"
 if [ -z "$version" ]; then
 echo "error: cpp_doc_push_script.sh: version (arg2) not specified"
   exit 1
@@ -35,7 +35,7 @@ echo "error: cpp_doc_push_script.sh: install_path (arg1) not specified"
 fi
 
 is_main_doc=false
-if [ "$version" == "main" ]; then
+if [ "$version" == "master" ]; then
   is_main_doc=true
 fi
 
@@ -56,7 +56,7 @@ sudo apt-get -y install doxygen
 # Generate ATen files
 pushd "${pt_checkout}"
 pip install -r requirements.txt
-time python -m tools.codegen.gen \
+time python -m torchgen.gen \
   -s aten/src/ATen \
   -d build/aten/src/ATen
 

.circleci/scripts/python_doc_push_script.sh

@@ -23,7 +23,7 @@ set -ex
 #       but since DOCS_INSTALL_PATH can be derived from DOCS_VERSION it's probably better to
 #       try and gather it first, just so we don't potentially break people who rely on this script
 # Argument 2: What version of the docs we are building.
-version="${2:-${DOCS_VERSION:-main}}"
+version="${2:-${DOCS_VERSION:-master}}"
 if [ -z "$version" ]; then
 echo "error: python_doc_push_script.sh: version (arg2) not specified"
   exit 1
@@ -38,7 +38,7 @@ echo "error: python_doc_push_script.sh: install_path (arg1) not specified"
 fi
 
 is_main_doc=false
-if [ "$version" == "main" ]; then
+if [ "$version" == "master" ]; then
   is_main_doc=true
 fi
 

.gitattributes

@@ -3,3 +3,4 @@
 .github/workflows/generated-*.yml linguist-generated=true
 .github/generated-* linguist-generated=true
 .github/scripts/gql_mocks.json linguist-generated=true
+third_party/LICENSES_BUNDLED.txt linguist-generated=true

.github/actions/checkout-pytorch/action.yml

@@ -6,6 +6,14 @@ inputs:
   no-sudo:
     description: If set to any value, don't use sudo to clean the workspace
     required: false
+  submodules:
+    description: Works as stated in actions/checkout, but the default value is recursive
+    required: false
+    default: recursive
+  fetch-depth:
+    description: Works as stated in actions/checkout, but the default value is 0
+    required: false
+    default: "0"
 
 runs:
   using: composite
@@ -27,6 +35,6 @@ runs:
       uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
       with:
         ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-        # deep clone, to allow use of git merge-base
-        fetch-depth: 0
-        submodules: recursive
+        # --depth=1 for speed, manually fetch history and other refs as necessary
+        fetch-depth: ${{ inputs.fetch-depth }}
+        submodules: ${{ inputs.submodules }}

.github/actions/setup-ssh/action.yml

@@ -14,3 +14,4 @@ runs:
       uses: seemethere/add-github-ssh-key@v1
       with:
         GITHUB_TOKEN: ${{ inputs.github-secret }}
+        activate-with-label: false

.github/merge_rules.json

@@ -3,8 +3,11 @@
       "name": "ONNX exporter",
       "patterns": [
          ".jenkins/caffe2/*",
+         "scripts/onnx/**",
          "docs/source/onnx.rst",
          "test/onnx/**",
+         "test/jit/test_export_modes.py",
+         "aten/src/ATen/core/interned_strings.h",
          "tools/onnx/**",
          "torch/_C/__init__.pyi.in",
          "torch/csrc/jit/passes/onnx.*",
@@ -20,7 +23,7 @@
    {
       "name": "NVFuser",
       "patterns": ["torch/csrc/jit/codegen/fuser/cuda/**", "torch/csrc/jit/codegen/cuda/**", "benchmarks/cpp/nvfuser/**"],
-      "approved_by": ["csarofeen", "ngimel"],
+      "approved_by": ["csarofeen", "ngimel", "jjsjann123"],
       "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
    },
    {
@@ -41,6 +44,62 @@
       "approved_by": ["linbinyu", "kit1980", "IvanKobzarev", "dreiss"],
       "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
    },
+   {
+      "name": "Linear Algebra",
+      "patterns": [
+         "aten/src/ATen/native/cuda/linalg/**",
+         "aten/src/ATen/LinalgBackend.h",
+         "aten/src/ATen/native/**/*LinearAlgebra*",
+         "docs/source/linalg.rst",
+         "torch/linalg/**",
+         "torch/_linalg_utils.py",
+         "torch/**/python_linalg_functions.*",
+         "torch/**/linalg.h",
+         "tools/autograd/templates/python_linalg_functions.cpp",
+         "test/test_linalg.py"
+      ],
+      "approved_by": ["nikitaved", "mruberry", "pearu", "Lezcano", "IvanYashchuk"],
+      "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
+   },
+   {
+      "name": "FFT",
+      "patterns": [
+         "aten/src/ATen/native/cuda/*FFT*.h",
+         "aten/src/ATen/native/SpectralOps.cpp",
+         "aten/src/ATen/native/mkl/SpectralOps.cpp",
+         "aten/src/ATen/native/cuda/SpectralOps.*",
+         "docs/source/fft.rst",
+         "torch/fft/**",
+         "torch/csrc/api/include/torch/fft.h",
+         "torch/**/python_fft_functions.*",
+         "tools/autograd/templates/python_fft_functions.cpp",
+         "test/cpp/api/fft.cpp"
+      ],
+      "approved_by": ["mruberry", "peterbell10"],
+      "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
+   },
+   {
+      "name": "Sparse",
+      "patterns": [
+         "benchmarks/sparse",
+         "c10/util/sparse_bitset.h",
+         "docs/source/sparse.rst",
+         "torch/**/sparse/**",
+         "torch/**/*sparse*",
+         "torch/optim/sparse*",
+         "torch/ao/nn/sparse/**",
+         "torch/utils/benchmark/**/*sparse*",
+         "aten/src/ATen/native/ao_sparse/**",
+         "aten/src/ATen/native/sparse/**",
+         "aten/src/ATen/**/*Sparse*",
+         "aten/src/ATen/*Sparse*",
+         "torch/_masked/**",
+         "test/*_masked*",
+         "test/**/*sparse*"
+      ],
+      "approved_by": ["nikitaved", "cpuhrsch", "pearu", "IvanYashchuk"],
+      "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
+   },
    {
       "name": "superuser",
       "patterns": ["*"],

.github/scripts/convert_lintrunner_annotations_to_github.py

@@ -0,0 +1,63 @@
+import json
+import subprocess
+import sys
+
+from enum import Enum
+from pathlib import Path
+from typing import NamedTuple, Optional
+
+# From: https://docs.github.com/en/rest/reference/checks
+class GitHubAnnotationLevel(str, Enum):
+    NOTICE = "notice"
+    WARNING = "warning"
+    FAILURE = "failure"
+
+
+class GitHubAnnotation(NamedTuple):
+    path: str
+    start_line: int
+    end_line: int
+    start_column: Optional[int]
+    end_column: Optional[int]
+    annotation_level: GitHubAnnotationLevel
+    message: str
+    title: Optional[str]
+    raw_details: Optional[str]
+
+PYTORCH_ROOT = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode('ascii').strip())
+
+annotations = []
+for line in sys.stdin:
+    lint_message = json.loads(line)
+
+    path = lint_message.get("path")
+    line = lint_message.get("line")
+
+
+    code = lint_message["code"]
+    severity = lint_message["severity"]
+    name = lint_message["name"]
+    description = lint_message.get("description")
+
+    # These fields are required by the GitHub API, but optional in lintrunner.
+    # If they don't exist, just skip.
+    if path is None or line is None:
+        print(f"No path/line for lint: ({code}) {name}", file=sys.stderr)
+        continue
+
+    # normalize path relative to git root
+    path = Path(path).relative_to(PYTORCH_ROOT)
+
+    annotations.append(GitHubAnnotation(
+        path=str(path),
+        start_line=int(line),
+        end_line=int(line),
+        start_column=None,
+        end_column=None,
+        annotation_level=GitHubAnnotationLevel.FAILURE,
+        message=description,
+        title=f"({code}) {name}",
+        raw_details=None,
+    )._asdict())
+
+print(json.dumps(annotations), flush=True)

.github/scripts/generate_binary_build_matrix.py

@@ -13,7 +13,7 @@
 from typing import Dict, List, Tuple, Optional
 
 
-CUDA_ARCHES = ["10.2", "11.3", "11.5", "11.6"]
+CUDA_ARCHES = ["10.2", "11.3", "11.6"]
 
 
 ROCM_ARCHES = ["4.5.2", "5.0"]
@@ -148,6 +148,9 @@ def generate_libtorch_matrix(os: str, abi_version: str,
             # matter
             gpu_arch_type = arch_type(arch_version)
             gpu_arch_version = "" if arch_version == "cpu" else arch_version
+            # ROCm builds without-deps failed even in ROCm runners; skip for now
+            if gpu_arch_type == "rocm" and "without-deps" in libtorch_variant:
+                continue
             ret.append(
                 {
                     "gpu_arch_type": gpu_arch_type,

.github/scripts/gitutils.py

@@ -200,7 +200,9 @@ def compute_branch_diffs(self, from_branch: str, to_branch: str) -> Tuple[List[s
                         # which creates a tracking problem
                         if (
                             "pytorch/pytorch" not in self.remote_url() or
-                            frc.commit_hash != "0a6a1b27a464ba5be5f587cce2ee12ab8c504dbf"
+                            frc.commit_hash not in {"0a6a1b27a464ba5be5f587cce2ee12ab8c504dbf",
+                                                    "6d0f4a1d545a8f161df459e8d4ccafd4b9017dbe",
+                                                    "edf909e58f06150f7be41da2f98a3b9de3167bca"}
                         ):
                             raise RuntimeError(f"Unexpected differences between {frc} and {toc}")
                     from_commits.remove(frc.commit_hash)
@@ -210,6 +212,15 @@ def compute_branch_diffs(self, from_branch: str, to_branch: str) -> Tuple[List[s
                 from_commits.remove(commit)
             for commit in to_values:
                 to_commits.remove(commit)
+        # Another HACK: Patch-id is not stable for commits with binary files or for big changes across commits
+        # I.e. cherry-picking those from one branch into another will change patchid
+        if "pytorch/pytorch" in self.remote_url():
+            for excluded_commit in {"8e09e20c1dafcdbdb45c2d1574da68a32e54a3a5",
+                                    "5f37e5c2a39c3acb776756a17730b865f0953432",
+                                    "b5222584e6d6990c6585981a936defd1af14c0ba"}:
+                if excluded_commit in from_commits:
+                    from_commits.remove(excluded_commit)
+
         return (from_commits, to_commits)
 
     def cherry_pick_commits(self, from_branch: str, to_branch: str) -> None: