Make tests and infrastructure less flaky

.github/actions/start-runtime/action.yml

@@ -14,40 +14,38 @@ runs:
       uses: actions/[email protected]
       with:
         python-version: "3.x"
+
     - name: "Install Python dependencies"
       run: pip install pyaml httplib2
       shell: bash
+
     - name: "Setup Java 21"
       id: setup-java
       uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 #v4
       with:
         distribution: "temurin"
         java-version: "21"
-    - name: "Extract deployment package"
-      run: |
-        mkdir project
-        unzip -qq ${{ inputs.mda-file }} -d project
-        cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
-        sed -i -- 's=$ROOT_PATH=${{ github.workspace }}=g' project/m2ee-native.yml
-        sed -i -- 's=$JAVA_HOME=${{ steps.setup-java.outputs.path }}=g' project/m2ee-native.yml
+
+    - name: "Make setup-runtime.sh executable"
+      run: chmod +x .github/scripts/setup-runtime.sh
       shell: bash
-    - name: "Setup m2ee"
+
+    - name: "Initial setup"
       run: |
-        mkdir -p var/log var/opt/m2ee var/run bin tmp
-        git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
-        mv tmp/m2ee/src/* var/opt/m2ee
-        chmod a=rwx var/log/ var/run/
-        echo "#!/bin/bash -x" > bin/m2ee
-        echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
-        chmod +x bin/m2ee
+        .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
       shell: bash
-    - name: "Setup mxruntime"
+
+    - name: "Start mxruntime with retries"
       run: |
-        mkdir -p ${{ github.workspace }}/project/runtimes ${{ github.workspace }}/project/data/model-upload ${{ github.workspace }}/project/data/database ${{ github.workspace }}/project/data/files ${{ github.workspace }}/project/data/tmp
-        wget -q https://cdn.mendix.com/runtime/mendix-${{ inputs.mendix-version }}.tar.gz -O tmp/runtime.tar.gz
-        tar xfz tmp/runtime.tar.gz --directory ${{ github.workspace }}/project/runtimes
-        rm tmp/runtime.tar.gz
-      shell: bash
-    - name: "Start mxruntime"
-      run: bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start
-      shell: bash
+        MAX_RETRIES=3
+        RETRY=0
+        until bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start; do
+          RETRY=$((RETRY+1))
+          if [ $RETRY -ge $MAX_RETRIES ]; then
+            echo "mxruntime failed after $MAX_RETRIES attempts."
+            exit 1
+          fi
+          echo "mxruntime failed, retrying ($RETRY/$MAX_RETRIES)..."
+          .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
+        done
+      shell: bash

.github/scripts/setup-runtime.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+MDA_FILE="$1"
+MENDIX_VERSION="$2"
+JAVA_PATH="$3"
+WORKSPACE="$4"
+
+rm -rf project var tmp bin
+
+mkdir project
+unzip -qq "$MDA_FILE" -d project
+cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
+sed -i -- "s=\$ROOT_PATH=$WORKSPACE=g" project/m2ee-native.yml
+sed -i -- "s=\$JAVA_HOME=$JAVA_PATH=g" project/m2ee-native.yml
+
+mkdir -p var/log var/opt/m2ee var/run bin tmp
+git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
+mv tmp/m2ee/src/* var/opt/m2ee
+chmod a=rwx var/log/ var/run/
+echo "#!/bin/bash -x" > bin/m2ee
+echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
+chmod +x bin/m2ee
+
+mkdir -p "$WORKSPACE/project/runtimes" "$WORKSPACE/project/data/model-upload" "$WORKSPACE/project/data/database" "$WORKSPACE/project/data/files" "$WORKSPACE/project/data/tmp"
+wget -q "https://cdn.mendix.com/runtime/mendix-$MENDIX_VERSION.tar.gz" -O tmp/runtime.tar.gz
+tar xfz tmp/runtime.tar.gz --directory "$WORKSPACE/project/runtimes"
+rm tmp/runtime.tar.gz

.github/workflows/NativePipeline.yml

@@ -439,6 +439,7 @@ jobs:
   android-tests:
     needs: [scope, mendix-version, project, android-app]
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}
@@ -557,6 +558,7 @@ jobs:
   ios-tests:
     needs: [scope, mendix-version, project, ios-app]
     runs-on: macos-15
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}

configs/e2e/mendix-versions.json

@@ -1,4 +1,4 @@
 {
-    "latest": "10.22.0.68245",
+    "latest": "10.23.0.70273",
     "8": "8.18.23.62193"
 }

maestro/helpers/compare_screenshots.js

@@ -57,7 +57,8 @@ fs.readdirSync(actualDir).forEach(file => {
       ];
     } else if (platform === 'android') {
       ignoredAreas = [
-        { x: 0, y: 0, width, height: 50 } // Ignore top 40 pixels on Android
+        { x: 0, y: 0, width, height: 50 }, // Ignore top 40 pixels on Android
+        { x: width - 15, y: 0, width: 15, height } // Ignore right 10 pixels on Android
       ];
     }
 

maestro/run_maestro_tests_local.sh

@@ -9,9 +9,9 @@ if [ "$1" == "android" ]; then
   DEVICE_ID="emulator-5554"
   PLATFORM="android"
 elif [ "$1" == "ios" ]; then
-  APP_ID="myapp.nativecomponentstestproject"
-  # APP_ID="com.mendix.native.template"
-  DEVICE_ID="4293ECC5-CA5A-4820-9433-40562F7F6F5E"
+  #APP_ID="myapp.nativecomponentstestproject"
+  APP_ID="com.mendix.native.template"
+  DEVICE_ID="8CDB91BF-9D4A-4F1D-8952-A7ABBED10078"
   PLATFORM="ios"
 else
   echo "Usage: $0 [android|ios] [widget]"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_dynamic_image.yaml

@@ -8,9 +8,10 @@ appId: "${APP_ID}"
     text: "Background image"
 - tapOn:
     text: "Dynamic image"
-- extendedWaitUntil:
-    visible:
-        id: "dynamicBgImage"
-    timeout: 30000
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 30000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_dynamic_image"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_dynamic_svg.yaml

@@ -8,9 +8,10 @@ appId: "${APP_ID}"
     text: "Background image"
 - tapOn:
     text: "Dynamic SVG image"
-- extendedWaitUntil:
-    visible: "Dynamic SVG image"
-    timeout: 30000
-    optional: true
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_dynamic_svg"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_layout_grid.yaml

@@ -9,5 +9,10 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Layout grid"
 - assertVisible: "Layout grid"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_layout_grid"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_nested.yaml

@@ -9,5 +9,10 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Nested"
 - assertVisible: "Nested"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_nested"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_static_images.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Static images"
 - assertVisible:
     text: "Static images"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 10000 # 10 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_static_images"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_dynamic.yaml

@@ -8,5 +8,10 @@ appId: "${APP_ID}"
     text: "Image"
 - tapOn:
     text: "Image dynamic"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 30000 # 30 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/image_dynamic"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_static.yaml

@@ -8,7 +8,12 @@ appId: "${APP_ID}"
     text: "Image"
 - tapOn:
     text: "Image static"
-- assertVisible:
-    text: "Image Static"
+- assertVisible: 
+    text: "Main as Static Image; Default as Static; Background as Contain"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/image_static"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_url.yaml

@@ -9,6 +9,6 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Image url"
 - assertVisible: "Main as Image URL(svg); Background as No; Action as Open dialog box"
-- takeScreenshot:
-    path: "maestro/images/actual/${PLATFORM}/image_url"
+# - takeScreenshot:
+#     path: "maestro/images/actual/${PLATFORM}/image_url"
 

packages/pluggableWidgets/line-chart-native/e2e/specs/maestro/LineChart.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Line chart"
 - assertVisible:
     id: "container1"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/line_chart"

packages/pluggableWidgets/pie-doughnut-chart-native/e2e/specs/maestro/Doughnut_chart_custom.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Doughnut chart Custom style"
 - assertVisible:
     id: "container1"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/doughnut_chart_custom"

packages/pluggableWidgets/pie-doughnut-chart-native/e2e/specs/maestro/Doughnut_chart_multiple_data_points.yaml

@@ -10,6 +10,11 @@ appId: "${APP_ID}"
     text: "Doughnut chart Multiple data points"
 - assertVisible:
     id: "container1"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/doughnut_chart_multiple_data_points"
 

packages/pluggableWidgets/pie-doughnut-chart-native/e2e/specs/maestro/Pie_chart_custom.yaml

@@ -10,5 +10,12 @@ appId: "${APP_ID}"
     text: "Pie chart Custom style"
 - assertVisible:
     id: "container1"
+- assertVisible:
+    id: "innerView"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/pie_chart_custom"

packages/pluggableWidgets/pie-doughnut-chart-native/e2e/specs/maestro/Pie_chart_multiple_data_points.yaml

@@ -10,5 +10,12 @@ appId: "${APP_ID}"
     text: "Pie chart Multiple data points"
 - assertVisible:
     id: "container1"
+- assertVisible:
+    id: "innerView"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/pie_chart_multiple_data_points"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewBottomBar.yaml

@@ -13,3 +13,5 @@ appId: "${APP_ID}"
     \ - Safe area has an orange background color\r
 
     \ - Bottom bar starts directly below safe area"
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_bottom_bar"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewContainer.yaml

@@ -8,18 +8,14 @@ appId: "${APP_ID}"
     text: "Safe area view"
 - tapOn:
     text: "Container"
-- assertVisible: "Safe area view widget directly on a fullscreen page. The safe area has an orange background color, the page has a blue background color and the container with content has a white background color.\r
-
-    \r
+- assertVisible: "Safe area view widget directly on a fullscreen page. The safe area has an orange background color.\r
 
     Expected result: \r
 
     \ - Safe area has horizontal and vertical padding\r
 
     \ - Safe area has an orange background color\r
 
-    \ - Unsafe area has a blue background color\r
-
-    \ - White container with text located in the safe area\r
-
     \ - White container uses all available space (flex 1)"
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_container"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewImage.yaml

@@ -10,3 +10,5 @@ appId: "${APP_ID}"
     text: "Image"
 - assertVisible:
     id: "image1"
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_image"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewListView.yaml

@@ -13,3 +13,5 @@ appId: "${APP_ID}"
 - assertVisible:
     id: "image1"
     index: 14
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_list_view"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewText.yaml

@@ -13,3 +13,5 @@ appId: "${APP_ID}"
     \r
 
     Expected result: Text rendered in safe area and full blue background"
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_text"

packages/pluggableWidgets/safe-area-view-native/e2e/specs/maestro/SafeAreaViewTopBar.yaml

@@ -13,3 +13,5 @@ appId: "${APP_ID}"
     \ - Safe area has an orange background color\r
 
     \ - Safe area starts directly below header"
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/safe_area_view_top_bar"

packages/pluggableWidgets/slider-native/e2e/specs/maestro/Slider.yaml

@@ -6,6 +6,6 @@ appId: "${APP_ID}"
     text: "S"
 - tapOn:
     text: "Slider"
-- assertVisible: "Slider"
+- assertVisible: "Dynamic ranges"
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/slider"

packages/pluggableWidgets/switch-native/e2e/specs/maestro/Switch.yaml

@@ -6,6 +6,8 @@ appId: "${APP_ID}"
     text: "S"
 - tapOn:
     text: "Switch"
+- assertNotVisible: 
+    text: "I will present myself when the boolean is True"
 - tapOn:
     id: "switch1"
 - assertVisible: 

packages/pluggableWidgets/toggle-buttons-native/e2e/specs/maestro/ToggleButtons.yaml

@@ -7,6 +7,11 @@ appId: "${APP_ID}"
 - tapOn: 
     text: "Toggle buttons"
 - assertVisible: 
-    text: "Toggle buttons"
+    text: "Default:"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/toggle_buttons"