Merge branch 'master' into issue-5826-fix

Author: me-no-dev

.github/ISSUE_TEMPLATE/Issue-report.yml

@@ -39,7 +39,8 @@ body:
       label: Version
       description: What version of Arduino ESP32 are you running? If possible, consider updating to the latest version.
       options:
-        - latest master
+        - latest master (checkout manually)
+        - latest development Release Candidate (RC-X)
         - v2.0.2
         - v2.0.1
         - v2.0.0
@@ -52,7 +53,7 @@ body:
     attributes:
      label: IDE Name
      description: What IDE are you using?
-     placeholder: eg. Arduino IDE, PlatformIO, IDF component...
+     placeholder: eg. Arduino IDE, PlatformIO, Sloeber...
     validations:
      required: true
   - type: input
@@ -101,8 +102,8 @@ body:
     id: sketch
     attributes:
       label: Sketch
-      description: Please provide your sketch/code which was run
-      placeholder: ex. related part of the code
+      description: Please provide full minimal sketch/code which can be run to reproduce your issue
+      placeholder: ex. Related part of the code to replicate the issue
       render: cpp
     validations:
      required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,23 @@
-*By completing this PR sufficiently, you help us to improve the quality of Release Notes*
+*By completing this PR sufficiently, you help us to review this Pull Request quicker and also help improve the quality of Release Notes*
 
 ### Checklist
-1. [ ] Please provide specific title of the PR describing the change, including the component name (eg. *„Update of Documentation link on Readme.md“*)
-2. [ ] Please provide related links (eg. Issue, other Project, submodule PR..)
-3. [ ] Please check [Contributing guide](https://docs.espressif.com/projects/arduino-esp32/en/latest/contributing.html)
+1. [ ] Please provide specific title of the PR describing the change, including the component name (*eg. „Update of Documentation link on Readme.md“*)
+2. [ ] Please provide related links (*eg. Issue which will be closed by this Pull Request*)
+3. [ ] Please **update relevant Documentation** if applicable
+4. [ ] Please check [Contributing guide](https://docs.espressif.com/projects/arduino-esp32/en/latest/contributing.html)
 
 *This entire section above can be deleted if all items are checked.*
 
 -----------
-## Summary
-Please describe your proposed PR and what it contains.
+## Description of Change
+Please describe your proposed Pull Request and it's impact.
 
-## Impact
-Please describe impact of your PR and it's function.
+## Tests scenarios
+Please describe on what Hardware and Software combinations you have tested this Pull Request and how.
+
+(*eg. I have tested my Pull Request on Arduino-esp32 core v2.0.2 with ESP32 and ESP32-S2 Board with this scenario*)
 
 ## Related links
 Please provide links to related issue, PRs etc.
+
+(*eg. Closes #number of issue*)

.github/scripts/install-platformio-esp32.sh

@@ -1,11 +1,9 @@
 #!/bin/bash
 
 export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32"
-PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git#feature/arduino-idf-master"
+PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git"
 
-XTENSA32_TOOLCHAIN_VERSION="8.4.0+2021r2-patch2"
-XTENSA32S2_TOOLCHAIN_VERSION="8.4.0+2021r2-patch2"
-RISCV_TOOLCHAIN_VERSION="8.4.0+2021r2-patch2"
+TOOLCHAIN_VERSION="8.4.0+2021r2-patch3"
 ESPTOOLPY_VERSION="~1.30100.0"
 ESPRESSIF_ORGANIZATION_NAME="espressif"
 
@@ -30,9 +28,12 @@ replace_script+="data['packages']['toolchain-xtensa-esp32']['owner']='$ESPRESSIF
 replace_script+="data['packages']['toolchain-xtensa-esp32s2']['owner']='$ESPRESSIF_ORGANIZATION_NAME';"
 replace_script+="data['packages']['toolchain-riscv32-esp']['owner']='$ESPRESSIF_ORGANIZATION_NAME';"
 # Update versions to use the upstream
-replace_script+="data['packages']['toolchain-xtensa-esp32']['version']='$XTENSA32_TOOLCHAIN_VERSION';"
-replace_script+="data['packages']['toolchain-xtensa-esp32s2']['version']='$XTENSA32S2_TOOLCHAIN_VERSION';"
-replace_script+="data['packages']['toolchain-riscv32-esp']['version']='$RISCV_TOOLCHAIN_VERSION';"
+replace_script+="data['packages']['toolchain-xtensa-esp32']['version']='$TOOLCHAIN_VERSION';"
+replace_script+="data['packages']['toolchain-xtensa-esp32s2']['version']='$TOOLCHAIN_VERSION';"
+replace_script+="data['packages']['toolchain-riscv32-esp']['version']='$TOOLCHAIN_VERSION';"
+# Add ESP32-S3 Toolchain
+replace_script+="data['packages'].update({'toolchain-xtensa-esp32s3':{'type':'toolchain','optional':True,'owner':'$ESPRESSIF_ORGANIZATION_NAME','version':'$TOOLCHAIN_VERSION'}});"
+replace_script+="data['packages']['toolchain-xtensa-esp32'].update({'optional':False});"
 # esptool.py may require an upstream version (for now platformio is the owner)
 replace_script+="data['packages']['tool-esptoolpy']['version']='$ESPTOOLPY_VERSION';"
 # Save results

.github/scripts/on-push.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+export ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp"
+
 function build(){
     local target=$1
     local fqbn=$2
@@ -58,11 +60,12 @@ fi
 
 SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    source ./.github/scripts/install-arduino-ide.sh
+    source ${SCRIPTS_DIR}/install-arduino-ide.sh
     source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
     FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
     FQBN_ESP32S2="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+    FQBN_ESP32S3="espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
     FQBN_ESP32C3="espressif:esp32:esp32c3:PartitionScheme=huge_app"
 
     SKETCHES_ESP32="\
@@ -76,11 +79,12 @@ if [ "$BUILD_PIO" -eq 0 ]; then
       $ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\
     "
 
-    build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
+    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
     build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
     build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
+    build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
 else
-    source ./${SCRIPTS_DIR}/install-platformio-esp32.sh
+    source ${SCRIPTS_DIR}/install-platformio-esp32.sh
     # PlatformIO ESP32 Test
     BOARD="esp32dev"
     OPTIONS="board_build.partitions = huge_app.csv"
@@ -96,6 +100,20 @@ else
     # build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
 
     python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32s2" --project-option="board_build.partitions = huge_app.csv"
+    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32c3" --project-option="board_build.partitions = huge_app.csv"
+
+    echo "Hacking in S3 support ..."
+    replace_script="import json; import os;"
+    replace_script+="fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+');"
+    replace_script+="data=json.load(fp);"
+    replace_script+="data['packages']['toolchain-xtensa-esp32']['optional']=True;"
+    replace_script+="data['packages']['toolchain-xtensa-esp32s3']['optional']=False;"
+    replace_script+="data['packages']['tool-esptoolpy']['owner']='tasmota';"
+    replace_script+="data['packages']['tool-esptoolpy']['version']='https://github.com/tasmota/esptool/releases/download/v3.3/esptool-3.3.zip';"
+    replace_script+="fp.seek(0);fp.truncate();json.dump(data, fp, indent=2);fp.close()"
+    python -c "$replace_script"
+
+    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32s3" --project-option="board_build.partitions = huge_app.csv"
 
     #build_pio_sketches "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries"
 fi

.github/scripts/on-release.sh

@@ -172,6 +172,7 @@ mkdir -p "$PKG_DIR/tools"
 # Copy all core files to the package folder
 echo "Copying files for packaging ..."
 cp -f  "$GITHUB_WORKSPACE/boards.txt"              "$PKG_DIR/"
+cp -f  "$GITHUB_WORKSPACE/package.json"            "$PKG_DIR/"
 cp -f  "$GITHUB_WORKSPACE/programmers.txt"         "$PKG_DIR/"
 cp -Rf "$GITHUB_WORKSPACE/cores"                   "$PKG_DIR/"
 cp -Rf "$GITHUB_WORKSPACE/libraries"               "$PKG_DIR/"

.github/scripts/sketch_utils.sh

@@ -7,15 +7,23 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <fqbn> <path-to-i
         return 1
     fi
 
-    ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     local ide_path=$1
     local usr_path=$2
     local fqbn=$3
     local sketch=$4
     local xtra_opts=$5
     local win_opts=$6
 
-    build_dir="$(dirname $sketch)/build"
+    ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
+    if [ -z "$ARDUINO_BUILD_DIR" ]; then
+        build_dir="$(dirname $sketch)/build"
+    else
+        build_dir="$ARDUINO_BUILD_DIR"
+    fi
+
+    echo $sketch
+
+    rm -rf "$build_dir"
     mkdir -p "$build_dir"
     mkdir -p "$ARDUINO_CACHE_DIR"
     $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \
@@ -32,13 +40,13 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <fqbn> <path-to-i
         $win_opts $xtra_opts "$sketch"
 }
 
-function count_sketches(){ # count_sketches <path> <target>
+function count_sketches(){ # count_sketches <path> [target]
     local path=$1
     local target=$2
 
-    if [ $# -lt 2 ]; then
+    if [ $# -lt 1 ]; then
       echo "ERROR: Illegal number of parameters"
-      echo "USAGE: ${0} count <path> <target>"
+      echo "USAGE: ${0} count <path> [target]"
     fi
 
     rm -rf sketches.txt
@@ -47,15 +55,15 @@ function count_sketches(){ # count_sketches <path> <target>
         return 0
     fi
 
-    local sketches=$(find $path -name *.ino)
+    local sketches=$(find $path -name *.ino | sort)
     local sketchnum=0
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
         if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
             continue
-        elif [[ -f "$sketchdir/.skip.$target" ]]; then
+        elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
             continue
         else
             echo $sketch >> sketches.txt

.github/scripts/tests_build.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+SCRIPTS_DIR="./.github/scripts"
+BUILD_CMD=""
+
+if [ $# -eq 3 ]; then
+    chunk_build=1
+elif [ $# -eq 2 ]; then
+    chunk_build=0
+else
+  echo "ERROR: Illegal number of parameters"
+  echo "USAGE:
+        ${0} <target> <sketch_dir>
+        ${0} <target> <chunk> <total_chunks>
+        "
+  exit 0
+fi
+
+target=$1
+
+case "$target" in
+    "esp32") fqbn="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
+    ;;
+    "esp32s2") fqbn="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+    ;;
+    "esp32c3") fqbn="espressif:esp32:esp32c3:PartitionScheme=huge_app"
+    ;;
+    "esp32s3") fqbn="espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
+    ;;
+esac
+
+if [ -z $fqbn ]; then
+  echo "Unvalid chip $1"
+  exit 0
+fi
+
+source ${SCRIPTS_DIR}/install-arduino-ide.sh
+source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
+
+args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH \"$fqbn\""
+
+if [ $chunk_build -eq 1 ]; then
+    chunk_index=$2
+    chunk_max=$3
+
+    if [ "$chunk_index" -gt "$chunk_max" ] &&  [ "$chunk_max" -ge 2 ]; then
+        chunk_index=$chunk_max
+    fi
+    BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
+    args+=" $target $PWD/tests $chunk_index $chunk_max"
+else
+    sketchdir=$2
+    BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
+    args+=" $PWD/tests/$sketchdir/$sketchdir.ino"
+fi
+
+${BUILD_CMD} ${args}
+

.github/scripts/tests_run.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+target=$1
+chunk_idex=$2
+chunks_num=$3
+
+SCRIPTS_DIR="./.github/scripts"
+COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
+
+source ${SCRIPTS_DIR}/install-arduino-ide.sh
+
+if [ "$chunks_num" -le 0 ]; then
+    echo "ERROR: Chunks count must be positive number"
+    return 1
+fi
+if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
+    echo "ERROR: Chunk index must be less than chunks count"
+    return 1
+fi
+
+set +e
+${COUNT_SKETCHES} $PWD/tests $target
+sketchcount=$?
+set -e
+sketches=$(cat sketches.txt)
+rm -rf sketches.txt
+
+chunk_size=$(( $sketchcount / $chunks_num ))
+all_chunks=$(( $chunks_num * $chunk_size ))
+if [ "$all_chunks" -lt "$sketchcount" ]; then
+    chunk_size=$(( $chunk_size + 1 ))
+fi
+
+start_index=0
+end_index=0
+if [ "$chunk_idex" -ge "$chunks_num" ]; then
+    start_index=$chunk_idex
+    end_index=$sketchcount
+else
+    start_index=$(( $chunk_idex * $chunk_size ))
+    if [ "$sketchcount" -le "$start_index" ]; then
+        echo "Skipping job"
+        return 0
+    fi
+
+    end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
+    if [ "$end_index" -gt "$sketchcount" ]; then
+        end_index=$sketchcount
+    fi
+fi
+
+start_num=$(( $start_index + 1 ))
+sketchnum=0
+
+for sketch in $sketches; do
+    sketchdir=$(dirname $sketch)
+    sketchdirname=$(basename $sketchdir)
+    sketchname=$(basename $sketch)
+    sketchnum=$(($sketchnum + 1))
+    if [ "$sketchnum" -le "$start_index" ] \
+    || [ "$sketchnum" -gt "$end_index" ]; then
+        continue
+    fi
+    echo ""
+    echo "Test for Sketch Index $(($sketchnum - 1)) - $sketchdirname"
+    pytest tests -k test_$sketchdirname --junit-xml=tests/$sketchdirname/$sketchdirname.xml
+    result=$?
+    if [ $result -ne 0 ]; then
+        return $result
+    fi
+done

.github/scripts/update-version.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# For reference: add tools for all boards by replacing one line in each board
+# "[board].upload.tool=esptool_py" to "[board].upload.tool=esptool_py\n[board].upload.tool.default=esptool_py\n[board].upload.tool.network=esp_ota"
+#cat boards.txt | sed "s/\([a-zA-Z0-9_\-]*\)\.upload\.tool\=esptool_py/\1\.upload\.tool\=esptool_py\\n\1\.upload\.tool\.default\=esptool_py\\n\1\.upload\.tool\.network\=esp_ota/"
+
+if [ ! $# -eq 3 ]; then
+	echo "Bad number of arguments: $#" >&2
+	echo "usage: $0 <major> <minor> <patch>" >&2
+	exit 1
+fi
+
+re='^[0-9]+$'
+if [[ ! $1 =~ $re ]] || [[ ! $2 =~ $re ]] || [[ ! $3 =~ $re ]] ; then
+	echo "error: Not a valid version: $1.$2.$3" >&2
+	echo "usage: $0 <major> <minor> <patch>" >&2
+	exit 1
+fi
+
+ESP_ARDUINO_VERSION_MAJOR="$1"
+ESP_ARDUINO_VERSION_MINOR="$2"
+ESP_ARDUINO_VERSION_PATCH="$3"
+ESP_ARDUINO_VERSION="$ESP_ARDUINO_VERSION_MAJOR.$ESP_ARDUINO_VERSION_MINOR.$ESP_ARDUINO_VERSION_PATCH"
+
+echo "New Arduino Version: $ESP_ARDUINO_VERSION"
+
+echo "Updating platform.txt..."
+cat platform.txt | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > __platform.txt && mv __platform.txt platform.txt
+
+echo "Updating package.json..."
+cat package.json | sed "s/.*\"version\":.*/  \"version\": \"$ESP_ARDUINO_VERSION\",/g" > __package.json && mv __package.json package.json
+
+echo "Updating cores/esp32/esp_arduino_version.h..."
+cat cores/esp32/esp_arduino_version.h | \
+sed "s/#define ESP_ARDUINO_VERSION_MAJOR.*/#define ESP_ARDUINO_VERSION_MAJOR   $ESP_ARDUINO_VERSION_MAJOR/g" | \
+sed "s/#define ESP_ARDUINO_VERSION_MINOR.*/#define ESP_ARDUINO_VERSION_MINOR   $ESP_ARDUINO_VERSION_MINOR/g" | \
+sed "s/#define ESP_ARDUINO_VERSION_PATCH.*/#define ESP_ARDUINO_VERSION_PATCH   $ESP_ARDUINO_VERSION_PATCH/g" > __esp_arduino_version.h && mv __esp_arduino_version.h cores/esp32/esp_arduino_version.h
+
+exit 0