Fixing E2E and UT flaky tests.

.github/workflows/ci_e2e_workflow.yml

@@ -58,7 +58,7 @@ jobs:
           }
 
           python -m pip install --upgrade pip
-          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
+          python -m pip install -U -e .[dev]
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
 
           # Retry a couple times to avoid certificate issue
@@ -76,7 +76,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.8 Tests
         if: matrix.python-version == 3.8
         env:
@@ -88,7 +88,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.9 Tests
         if: matrix.python-version == 3.9
         env:
@@ -100,7 +100,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.10 Tests
         if: matrix.python-version == 3.10
         env:
@@ -112,7 +112,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.11 Tests
         if: matrix.python-version == 3.11
         env:
@@ -124,7 +124,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Codecov
         uses: codecov/[email protected]
         with:

.github/workflows/ci_ut_workflow.yml

@@ -55,7 +55,7 @@ jobs:
         }
 
         python -m pip install --upgrade pip
-        python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
+        python -m pip install -U -e .[dev]
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
 
         # Retry a couple times to avoid certificate issue
@@ -66,7 +66,7 @@ jobs:
       env:
         AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} # needed for installing azure-functions-durable while running setup.py
       run: |
-        python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
+        python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
     - name: Codecov
       uses: codecov/codecov-action@v3
       with:

tests/endtoend/test_blob_functions.py

@@ -11,7 +11,6 @@ class TestBlobFunctions(testutils.WebHostTestCase):
     def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'blob_functions'
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_str(self):
         r = self.webhost.request('POST', 'put_blob_str', data='test-data')
         self.assertEqual(r.status_code, 200)
@@ -25,7 +24,6 @@ def test_blob_io_str(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, 'test-data')
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_large_str(self):
         large_string = 'DummyDataDummyDataDummyData' * 1024 * 1024  # 27 MB
 
@@ -41,7 +39,6 @@ def test_blob_io_large_str(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, large_string)
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_bytes(self):
         r = self.webhost.request('POST', 'put_blob_bytes',
                                  data='test-dată'.encode('utf-8'))
@@ -56,7 +53,6 @@ def test_blob_io_bytes(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, 'test-dată')
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_large_bytes(self):
         large_string = 'DummyDataDummyDataDummyData' * 1024 * 1024  # 27 MB
 
@@ -73,7 +69,6 @@ def test_blob_io_large_bytes(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, large_string)
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_filelike(self):
         r = self.webhost.request('POST', 'put_blob_filelike')
         self.assertEqual(r.status_code, 200)
@@ -83,7 +78,6 @@ def test_blob_io_filelike(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, 'filelike')
 
-    @testutils.retryable_test(3, 5)
     def test_blob_io_return(self):
         r = self.webhost.request('POST', 'put_blob_return')
         self.assertEqual(r.status_code, 200)
@@ -92,7 +86,6 @@ def test_blob_io_return(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, 'FROM RETURN')
 
-    @testutils.retryable_test(3, 5)
     def test_blob_trigger(self):
         data = "DummyData"
 
@@ -126,7 +119,6 @@ def test_blob_trigger(self):
                 if try_no == max_retries - 1:
                     raise
 
-    @testutils.retryable_test(3, 5)
     def test_blob_trigger_with_large_content(self):
         data = 'DummyDataDummyDataDummyData' * 1024 * 1024  # 27 MB
 

tests/endtoend/test_blueprint_functions.py

@@ -10,7 +10,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
             'functions_in_blueprint_only'
 
-    @testutils.retryable_test(3, 5)
     def test_function_in_blueprint_only(self):
         r = self.webhost.request('GET', 'default_template')
         self.assertTrue(r.ok)
@@ -22,7 +21,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
             'functions_in_both_blueprint_functionapp'
 
-    @testutils.retryable_test(3, 5)
     def test_functions_in_both_blueprint_functionapp(self):
         r = self.webhost.request('GET', 'default_template')
         self.assertTrue(r.ok)
@@ -37,7 +35,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
             'multiple_function_registers'
 
-    @testutils.retryable_test(3, 5)
     def test_function_in_blueprint_only(self):
         r = self.webhost.request('GET', 'return_http')
         self.assertEqual(r.status_code, 404)
@@ -49,7 +46,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
             'only_blueprint'
 
-    @testutils.retryable_test(3, 5)
     def test_only_blueprint(self):
         """Test if the default template of Http trigger in Python
         Function app

tests/endtoend/test_cosmosdb_functions.py

@@ -49,7 +49,6 @@ def test_cosmosdb_trigger(self):
             else:
                 break
 
-    @testutils.retryable_test(3, 5)
     def test_cosmosdb_input(self):
         time.sleep(5)
         data = str(round(time.time()))

tests/endtoend/test_dependency_isolation_functions.py

@@ -52,15 +52,13 @@ def tearDownClass(self):
     def get_script_dir(cls):
         return cls.project_root
 
-    @testutils.retryable_test(3, 5)
     def test_dependency_function_should_return_ok(self):
         """The common scenario of general import should return OK in any
         circumstances
         """
         r: Response = self.webhost.request('GET', 'report_dependencies')
         self.assertTrue(r.ok)
 
-    @testutils.retryable_test(3, 5)
     def test_feature_flag_is_turned_on(self):
         """Since passing the feature flag PYTHON_ISOLATE_WORKER_DEPENDENCIES to
         the host, the customer's function should also be able to receive it
@@ -70,7 +68,6 @@ def test_feature_flag_is_turned_on(self):
         flag_value = environments['PYTHON_ISOLATE_WORKER_DEPENDENCIES']
         self.assertEqual(flag_value, '1')
 
-    @testutils.retryable_test(3, 5)
     def test_working_directory_resolution(self):
         """Check from the dependency manager and see if the current working
         directory is resolved correctly
@@ -88,7 +85,6 @@ def test_working_directory_resolution(self):
         is_envvar_true(PYAZURE_INTEGRATION_TEST),
         'Integration test expects dependencies derived from core tools folder'
     )
-    @testutils.retryable_test(3, 5)
     def test_paths_resolution(self):
         """Dependency manager requires paths to be resolved correctly before
         switching to customer's modules. This test is to ensure when the app
@@ -112,7 +108,6 @@ def test_paths_resolution(self):
             ).lower()
         )
 
-    @testutils.retryable_test(3, 5)
     def test_loading_libraries_from_customers_package(self):
         """Since the Python now loaded the customer's dependencies, the
         libraries version should match the ones in
@@ -170,7 +165,6 @@ def tearDownClass(self):
     def get_script_dir(cls):
         return cls.project_root
 
-    @testutils.retryable_test(3, 5)
     def test_loading_libraries_from_customers_package(self):
         r: Response = self.webhost.request('GET', 'report_dependencies')
         libraries = r.json()['libraries']
@@ -219,7 +213,6 @@ def tearDownClass(self):
     def get_script_dir(cls):
         return cls.project_root
 
-    @testutils.retryable_test(3, 5)
     def test_loading_libraries_from_customers_package(self):
         r: Response = self.webhost.request('GET', 'report_dependencies')
         libraries = r.json()['libraries']

tests/endtoend/test_eventgrid_functions.py

@@ -27,7 +27,6 @@ def eventgrid_webhook_request(self, meth, funcname, *args, **kwargs):
         return request_method(url, *args, params=params, headers=headers,
                               **kwargs)
 
-    @testutils.retryable_test(3, 5)
     @unittest.skip("Run locally. Running on Azure fails with 401/403 as the"
                    "host does not pick up the SecretKey from the"
                    "azure_functions_worker.testutils.py.SECRETS_TEMPLATE and"
@@ -92,7 +91,6 @@ def test_eventgrid_trigger(self):
             else:
                 break
 
-    @testutils.retryable_test(3, 5)
     def test_eventgrid_output_binding(self):
         """test event_grid output binding
 

tests/endtoend/test_eventhub_batch_functions.py

@@ -22,7 +22,6 @@ class TestEventHubFunctions(testutils.WebHostTestCase):
     def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions'
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_multiple(self):
         NUM_EVENTS = 3
         all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
@@ -65,7 +64,6 @@ def test_eventhub_multiple(self):
         finally:
             self._cleanup(old_partition_key)
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_multiple_with_metadata(self):
         # Generate a unique event body for EventHub event
         # Record the start_time and end_time for checking event enqueue time
@@ -165,7 +163,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' / \
             'eventhub_batch_functions_stein'
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_multiple(self):
         NUM_EVENTS = 3
         all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
@@ -199,7 +196,6 @@ def test_eventhub_multiple(self):
 
         self.assertDictEqual(all_row_keys_seen, row_keys_seen)
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_multiple_with_metadata(self):
         # Generate a unique event body for EventHub event
         # Record the start_time and end_time for checking event enqueue time

tests/endtoend/test_eventhub_functions.py

@@ -22,7 +22,6 @@ class TestEventHubFunctions(testutils.WebHostTestCase):
     def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'eventhub_functions'
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_trigger(self):
         # Generate a unique event body for the EventHub event
         data = str(round(time.time()))
@@ -47,7 +46,6 @@ def test_eventhub_trigger(self):
         # Check if the event body matches the initial data
         self.assertEqual(response, doc)
 
-    @testutils.retryable_test(3, 5)
     def test_eventhub_trigger_with_metadata(self):
         # Generate a unique event body for EventHub event
         # Record the start_time and end_time for checking event enqueue time

tests/endtoend/test_http_functions.py

@@ -139,7 +139,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'http_functions' / \
                                             'common_libs_functions'
 
-    @testutils.retryable_test(3, 5)
     def test_numpy(self):
         r = self.webhost.request('GET', 'numpy_func',
                                  timeout=REQUEST_TIMEOUT_SEC)
@@ -148,46 +147,40 @@ def test_numpy(self):
 
         self.assertEqual(r.content.decode("UTF-8"), res)
 
-    @testutils.retryable_test(3, 5)
     def test_requests(self):
         r = self.webhost.request('GET', 'requests_func',
                                  timeout=10)
 
         self.assertTrue(r.ok)
         self.assertEqual(r.content.decode("UTF-8"), 'req status code: 200')
 
-    @testutils.retryable_test(3, 5)
     def test_pandas(self):
         r = self.webhost.request('GET', 'pandas_func',
                                  timeout=REQUEST_TIMEOUT_SEC)
 
         self.assertIn("two-dimensional",
                       r.content.decode("UTF-8"))
 
-    @testutils.retryable_test(3, 5)
     def test_sklearn(self):
         r = self.webhost.request('GET', 'sklearn_func',
                                  timeout=REQUEST_TIMEOUT_SEC)
 
         self.assertIn("First 5 records of array:",
                       r.content.decode("UTF-8"))
 
-    @testutils.retryable_test(3, 5)
     def test_opencv(self):
         r = self.webhost.request('GET', 'opencv_func',
                                  timeout=REQUEST_TIMEOUT_SEC)
 
         self.assertIn("opencv version:",
                       r.content.decode("UTF-8"))
 
-    @testutils.retryable_test(3, 5)
     def test_dotenv(self):
         r = self.webhost.request('GET', 'dotenv_func',
                                  timeout=REQUEST_TIMEOUT_SEC)
 
         self.assertEqual(r.content.decode("UTF-8"), "found")
 
-    @testutils.retryable_test(3, 5)
     def test_plotly(self):
         r = self.webhost.request('GET', 'plotly_func',
                                  timeout=REQUEST_TIMEOUT_SEC)

tests/endtoend/test_table_functions.py

@@ -13,7 +13,6 @@ class TestTableFunctions(testutils.WebHostTestCase):
     def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'table_functions'
 
-    @testutils.retryable_test(3, 5)
     def test_table_bindings(self):
         out_resp = self.webhost.request('POST', 'table_out_binding')
         self.assertEqual(out_resp.status_code, 200)
@@ -46,7 +45,6 @@ def get_script_dir(cls):
         return testutils.E2E_TESTS_FOLDER / 'table_functions' / \
                                             'table_functions_stein'
 
-    @testutils.retryable_test(3, 5)
     def test_table_bindings(self):
         out_resp = self.webhost.request('POST', 'table_out_binding')
         self.assertEqual(out_resp.status_code, 200)

tests/unittests/test_enable_debug_logging_functions.py

@@ -4,8 +4,6 @@
 import os
 from unittest.mock import patch
 
-import pytest
-
 from tests.utils import testutils
 from azure_functions_worker.constants import PYTHON_ENABLE_DEBUG_LOGGING
 from tests.utils.testutils import TESTS_ROOT, remove_path
@@ -44,7 +42,6 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
-    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_enabled(self):
         """
         Verify when cx debug logging is enabled, cx function debug logs
@@ -82,7 +79,6 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
-    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_disabled(self):
         """
         Verify when cx debug logging is disabled, cx function debug logs
@@ -129,7 +125,6 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
-    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_filtered(self):
         """
         Verify when cx debug logging is enabled and host logging level