Update base for Update on "[ExecuTorch][#10375] Add extension.BundledModule to Wrap extension.Module with Bundled Program Logic"

#10375

# Context

This issue is a step of #9638.

In #9638, we want to have `extension.Module` as the single source of implementation in `pybindings`, which means that `pybindings.PyModule` should use `extension.Module` rather than its own `pybindings.Module`.

The issue is that `pybindings.PyModule` is dependent on the `method` getter from `pybindings.Module`, which `extension.Module` do not have. Since we don't want to expose `method` getter in `extension.Module`, we have to protect the getter, wrap the functions that is dependent on it and use the protected getter there, ultimately decouple `pybindings` from a `method` getter.

# Proposal

Now that we have a protected `method` getter, we can introduce a `extension.BundledModule`, a child class inheriting `extension.Module` which wraps up bundled program logic that is dependent on the `method` getter.

Differential Revision: [D73564125](https://our.internmc.facebook.com/intern/diff/D73564125/)

[ghstack-poisoned]

Author: zhenyan-zhang-meta

.ci/scripts/check_c10_sync.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eux
+ls pytorch/.git || git clone https://github.com/pytorch/pytorch.git
+pytorch_pin="$(< .ci/docker/ci_commit_pins/pytorch.txt)"
+pushd pytorch
+git checkout "$pytorch_pin"
+popd
+"$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/c10 pytorch/c10

.ci/scripts/compare_dirs.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eux
+
+# Check if dir1's files are also found in dir2 with the same
+# contents. Exempt files named BUCK, CMakeLists.txt, TARGETS, or
+# targets.bzl.
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 dir1 dir2" >&2
+    exit 1
+fi
+dir1="$1"
+dir2="$2"
+
+if [ ! -d "$dir1" ] || [ ! -d "$dir2" ]; then
+    echo "Error: Both directories must exist" >&2
+    exit 1
+fi
+
+exit_status=0
+while IFS= read -r -d '' file; do
+    base=$(basename "$file")
+    case "$base" in
+        "BUCK"|"CMakeLists.txt"|"TARGETS"|"targets.bzl")
+            continue
+            ;;
+    esac
+    # Construct the corresponding path in the second directory
+    file2="$dir2/${file#$dir1/}"
+    # Check if the corresponding file exists in the second directory
+    if [ ! -f "$file2" ]; then
+        echo "Error: File '$file' found in '$dir1' but not found in '$dir2'" >&2
+        exit 1
+    fi
+    # Compare the contents of the two files using diff
+    set +ex
+    differences=$(diff -u -p "$file" "$file2")
+    set -e # leave x off
+    # If there are any differences, print an error message and exit with failure status
+    if [ -n "$differences" ]; then
+        echo "Error: Mismatch detected in file '$file':" >&2
+        echo "$differences" >&2
+        exit_status=1
+    fi
+    set -x
+done < <(find "$dir1" -type f -print0)
+
+exit $exit_status

.github/workflows/_unittest.yml

@@ -33,7 +33,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: ${{ inputs.docker-image }}
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -45,7 +45,7 @@ jobs:
     with:
       runner: macos-m1-stable
       python-version: '3.11'
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         set -eux

.github/workflows/android-perf.yml

@@ -167,7 +167,7 @@ jobs:
     with:
       runner: linux.2xlarge.memory
       docker-image: executorch-ubuntu-22.04-qnn-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       timeout: 60
       upload-artifact: android-models
       upload-artifact-to-s3: true

.github/workflows/apple-perf.yml

@@ -167,7 +167,7 @@ jobs:
       # NB: Need to use our AWS MacOS runner to upload large models to S3
       runner: macos-m1-stable
       python-version: '3.11'
-      submodules: 'true'
+      submodules: 'recursive'
       timeout: 60
       upload-artifact: ios-models
       upload-artifact-to-s3: true

.github/workflows/check-c10-sync.yml

@@ -0,0 +1,27 @@
+name: check-c10-sync
+
+on:
+  pull_request:
+    paths:
+      - .ci/docker/ci_commit_pins/pytorch.txt
+      - .ci/scripts/compare_dirs.sh
+      - runtime/core/portable_type/c10/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
+  cancel-in-progress: true
+
+jobs:
+  check-c10-sync:
+    permissions:
+      id-token: write
+      contents: read
+    name: check-c10-sync
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Clone PyTorch
+        run: |
+          .ci/scripts/check_c10_sync.sh

.github/workflows/doc-build.yml

@@ -41,7 +41,7 @@ jobs:
       job-name: Build doc
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12-android
-      submodules: 'true'
+      submodules: 'recursive'
       repository: pytorch/executorch
       upload-artifact: docs
       timeout: 90

.github/workflows/lint.yml

@@ -23,7 +23,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-linter
-      submodules: 'true'
+      submodules: 'recursive'
       fetch-depth: 0
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90

.github/workflows/periodic.yml

@@ -50,7 +50,7 @@ jobs:
     with:
       runner: ${{ matrix.runner }}
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ matrix.timeout }}
       script: |

.github/workflows/pull.yml

@@ -24,7 +24,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-gcc9
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -66,7 +66,7 @@ jobs:
     with:
       runner: ${{ matrix.runner }}
       docker-image: ${{ matrix.docker-image }}
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -119,7 +119,7 @@ jobs:
     with:
       runner: ${{ matrix.runner }}
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -165,7 +165,7 @@ jobs:
     with:
       runner: ${{ matrix.runner }}
       docker-image: ${{ matrix.docker-image }}
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 900
       script: |
@@ -197,7 +197,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12-android
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -222,7 +222,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -246,7 +246,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -270,7 +270,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -301,7 +301,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -331,7 +331,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -354,7 +354,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -382,7 +382,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-gcc9
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -418,7 +418,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -489,7 +489,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-arm-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -528,7 +528,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-qnn-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 900
       script: |
@@ -563,7 +563,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-qnn-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 180
       script: |
@@ -596,7 +596,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-qnn-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 180
       script: |
@@ -618,7 +618,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -645,7 +645,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -672,7 +672,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -699,7 +699,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -726,7 +726,7 @@ jobs:
     with:
       runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-mediatek-sdk
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
@@ -747,7 +747,7 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-gcc9
-      submodules: 'true'
+      submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |