Merge pull request #108 from per1234/support-external-python-deps

Restore support for external Python package platform dependencies

Author: per1234

.github/workflows/test-integration.yml

@@ -6,19 +6,18 @@ on:
       - '.github/workflows/test-integration.yml'
       - '.github/workflows/testdata/**'
       - 'action.yml'
-      - 'action-setup.sh'
       - 'compilesketches/**'
 
   push:
     paths:
       - '.github/workflows/test-integration.yml'
       - '.github/workflows/testdata/**'
       - 'action.yml'
-      - 'action-setup.sh'
       - 'compilesketches/**'
 
 env:
   SKETCHES_REPORTS_PATH: sketches-reports
+  TESTDATA_PLATFORMS_PATH: .github/workflows/testdata/platforms
   TESTDATA_SKETCHES_PATH: .github/workflows/testdata/sketches
   TESTDATA_REPORTS_PATH:  .github/workflows/testdata/reports
 
@@ -203,6 +202,64 @@ jobs:
           sketch-paths: |
             - examples/Sweep
 
+  python-package-dependency:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout local repo
+        uses: actions/checkout@v3
+
+      - name: Install Python package dependency
+        run: |
+          pip install \
+            --ignore-installed \
+            --user \
+            cowsay
+
+      - name: Run action with board that has external Python package dependency
+        # Use action from local path
+        uses: ./
+        with:
+          platforms: |
+            - name: arduino:avr
+            - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PythonPackageDependent
+              name: PythonPackageDependent:avr
+          fqbn: PythonPackageDependent:avr:package_dependent
+          libraries: |
+            []
+          sketch-paths: |
+            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum
+
+
+  # Targeted testing for ESP32 boards platform support.
+  pyserial-dependency:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout local repo
+        uses: actions/checkout@v3
+
+      - name: Install pyserial
+        run: |
+          # Use of pip3 and omission of recommended flags done to reproduce established use pattern:
+          # https://github.com/arduino-libraries/ArduinoIoTCloud/blob/1.11.0/.github/workflows/compile-examples.yml#L206
+          pip3 install pyserial
+
+      - name: Run action with board that has pyserial dependency
+        # Use action from local path
+        uses: ./
+        with:
+          platforms: |
+            - name: arduino:avr
+            - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PyserialDependent
+              name: PyserialDependent:avr
+          fqbn: PyserialDependent:avr:pyserial_dependent
+          libraries: |
+            []
+          sketch-paths: |
+            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum
+
+
   check-sketches-reports:
     needs: all-inputs
     runs-on: ubuntu-latest

.github/workflows/testdata/platforms/PyserialDependent/boards.txt

@@ -0,0 +1,8 @@
+pyserial_dependent.name=pyserial Dependent Board
+pyserial_dependent.upload.maximum_size=32256
+pyserial_dependent.upload.maximum_data_size=2048
+pyserial_dependent.build.mcu=atmega328p
+pyserial_dependent.build.f_cpu=16000000L
+pyserial_dependent.build.board=FOO
+pyserial_dependent.build.core=arduino:arduino
+pyserial_dependent.build.variant=arduino:standard

.github/workflows/testdata/platforms/PyserialDependent/platform.txt

@@ -0,0 +1,5 @@
+name=pyserial Dependent Platform
+version=0.0.0
+# python3 is used to parallel ESP32 platform configuration:
+# https://github.com/espressif/arduino-esp32/blob/2.0.7/platform.txt#L220
+recipe.hooks.prebuild.1.pattern=python3 -c 'import serial'

.github/workflows/testdata/platforms/PythonPackageDependent/boards.txt

@@ -0,0 +1,8 @@
+package_dependent.name=External Python Package Dependent Board
+package_dependent.upload.maximum_size=32256
+package_dependent.upload.maximum_data_size=2048
+package_dependent.build.mcu=atmega328p
+package_dependent.build.f_cpu=16000000L
+package_dependent.build.board=FOO
+package_dependent.build.core=arduino:arduino
+package_dependent.build.variant=arduino:standard

.github/workflows/testdata/platforms/PythonPackageDependent/dependent.py

@@ -0,0 +1,2 @@
+# Import external package to verify Python package dependencies can be installed by user into runner environment.
+import cowsay  # noqa: F401

.github/workflows/testdata/platforms/PythonPackageDependent/platform.txt

@@ -0,0 +1,3 @@
+name=External Python Package Dependent Platform
+version=0.0.0
+recipe.hooks.prebuild.1.pattern=python "{runtime.platform.path}/dependent.py"

README.md

@@ -35,6 +35,7 @@ This action checks whether [Arduino](https://www.arduino.cc/) sketches compile a
     - [How it works](#how-it-works)
   - [`enable-warnings-report`](#enable-warnings-report)
 - [Example usage](#example-usage)
+- [Additional resources](#additional-resources)
 
 <!-- tocstop -->
 
@@ -226,3 +227,10 @@ Set to `true` to cause the action to record the compiler warning count for each
       - name: Stepper
         version: 1.1.3
 ```
+
+## Additional resources
+
+- [Introductory article about **arduino/compile-sketches**](https://blog.arduino.cc/2021/04/09/test-your-arduino-projects-with-github-actions/)
+- [Frequently asked questions about **arduino/compile-sketches**](docs/FAQ.md#frequently-asked-questions)
+- [**GitHub Actions** documentation](https://docs.github.com/actions/learn-github-actions/understanding-github-actions)
+- [Discuss or request assistance on **Arduino Forum**](https://forum.arduino.cc/)

action.yml

@@ -49,6 +49,13 @@ inputs:
 runs:
   using: composite
   steps:
+    # User installations of external Python package platform dependencies will be located here.
+    - name: Get system Python "user site-packages" path
+      id: system-user-site-packages
+      shell: bash
+      run: |
+        echo "path=$(python -m site --user-site)" >> $GITHUB_OUTPUT
+
     - name: Install Python
       uses: actions/[email protected]
       with:
@@ -58,14 +65,32 @@ runs:
       shell: bash
       run: |
         pipx install \
+          --python "$(which python)" \
           poetry==1.4.0
 
     - name: Install Python Dependencies
       shell: bash
       working-directory: ${{ github.action_path }}
       run: |
         poetry install \
-          --only main
+          --only main,external
+
+    - name: Make user-installed Python packages available to platforms
+      shell: bash
+      working-directory: ${{ github.action_path }}
+      run: |
+        readonly PYTHON_ENVIRONMENT_PATH="$(
+          poetry env info \
+            --path
+        )"
+        readonly VENV_SITE_PACKAGES_PATH="$(
+          poetry run \
+            python -c \
+              'import site; print(site.getsitepackages()[0])'
+        )"
+        echo \
+          "${{ steps.system-user-site-packages.outputs.path }}" > \
+          "${VENV_SITE_PACKAGES_PATH}/system-user-site-packages.pth"
 
     - name: Run script
       shell: bash
@@ -84,4 +109,4 @@ runs:
       working-directory: ${{ github.action_path }}
       run: |
         poetry run \
-          python "${{ github.action_path }}/compilesketches/compilesketches.py"
+          python compilesketches/compilesketches.py

docs/FAQ.md

@@ -0,0 +1,60 @@
+# Frequently Asked Questions
+
+## How can I install dependencies of a boards platform?
+
+### Managed Dependencies
+
+The Arduino **Boards Manager** system installs tool dependencies along with a platform. When you specify a [**Boards Manager**-sourced platform dependency](../README.md#boards-manager) via the action's [`platforms` input](../README.md#platforms) the managed platform dependencies are installed automatically.
+
+If an alternative [platform dependency source](../README.md#supported-platform-sources) is used this automatic tool dependency installation does not occur. The convenient way to install the tool dependencies in this case is to install a **Boards Manager**-sourced platform that has a dependency on the required tools in addition to the target platform from the alternative source.
+
+---
+
+**Example:**
+
+```yaml
+- uses: arduino/compile-sketches@v1
+  with:
+    platforms: |
+      # Use Boards Manager to install the latest release of the platform to get the toolchain.
+      - name: arduino:avr
+      # Overwrite the Boards Manager installation with the development version of the platform.
+      - source-url: https://github.com/arduino/ArduinoCore-avr.git
+        name: arduino:avr
+```
+
+---
+
+### External Dependencies
+
+Arduino boards platforms typically bundle all dependencies. However, there are some platforms that require the user to manually install dependencies on their system in order to use the platform.
+
+The **arduino/compile-sketches** action runs in the same environment as the rest of the steps of the [workflow job](https://docs.github.com/actions/using-jobs/using-jobs-in-a-workflow), which means you can simply perform the dependency installation in a prior [step](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps) of the job.
+
+---
+
+**Example:**
+
+```yaml
+- run: <some dependency installation command>
+- uses: arduino/compile-sketches@v1
+```
+
+---
+
+#### Python Packages
+
+The **arduino/compile-sketches** action uses a Python [virtual environment](https://docs.python.org/glossary.html#term-virtual-environment). In order to enable user installation of Python [package](https://docs.python.org/glossary.html#term-package) dependencies of boards platforms, the packages installed in the "[user site-packages](https://peps.python.org/pep-0370/)" folder are included in this virtual environment.
+
+In order to be certain your installation of a package dependency will be available to the platform, add the [`--ignore-installed`](https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-ignore-installed) and [`--user`](https://pip.pypa.io/en/stable/cli/pip_install/#install-user) flags to the [**pip**](https://pip.pypa.io/) command used to install the package.
+
+---
+
+**Example:**
+
+```yaml
+- run: pip install --ignore-installed --user pyserial
+- uses: arduino/compile-sketches@v1
+```
+
+---

poetry.lock

@@ -18,6 +18,14 @@ pytest-mock = "3.10.0"
 flake8 = "5.0.4"
 pep8-naming = "0.13.3"
 
+# Provided only for use by boards platforms
+# NOTE: This group is a temporary workaround that will be removed at the 2.0.0 release of the action.
+[tool.poetry.group.external]
+optional = true
+
+[tool.poetry.group.external.dependencies]
+pyserial = "3.5"
+
 [tool.pytest.ini_options]
 filterwarnings = [
   "error",