bug(test-deploy): Add retries for deploy tests (#1392)

Author: karangattu

tests/playwright/deploys/express-accordion/test_deploys_express_accordion.py

@@ -1,11 +1,18 @@
+import pytest
 from controls import Accordion
 from playwright.sync_api import Page
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("shiny_express_accordion")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_express_accordion(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-dataframe/test_deploys_express_dataframe.py

@@ -1,11 +1,18 @@
+import pytest
 from controls import OutputDataFrame
 from playwright.sync_api import Page
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("shiny-express-dataframe")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_express_dataframe_deploys(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-folium/test_deploys_express_folium.py

@@ -1,10 +1,17 @@
+import pytest
 from playwright.sync_api import Page, expect
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("shiny-express-folium")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_folium_map(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-page_default/test_deploys_express_page_default.py

@@ -1,13 +1,20 @@
+import pytest
 from controls import LayoutNavsetTab
 from playwright.sync_api import Page, expect
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 TIMEOUT = 2 * 60 * 1000
 
 app_url = create_deploys_app_url_fixture("shiny_express_page_default")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_page_default(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-page_fillable/test_deploys_express_page_fillable.py

@@ -1,11 +1,18 @@
+import pytest
 from controls import Card, OutputTextVerbatim
 from playwright.sync_api import Page
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("express_page_fillable")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_express_page_fillable(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-page_fluid/test_deploys_express_page_fluid.py

@@ -1,11 +1,18 @@
+import pytest
 from controls import Card, OutputTextVerbatim
 from playwright.sync_api import Page
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("express_page_fluid")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_express_page_fluid(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/express-page_sidebar/test_deploys_express_page_sidebar.py

@@ -1,11 +1,18 @@
+import pytest
 from controls import OutputTextVerbatim, Sidebar
 from playwright.sync_api import Page
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("express_page_sidebar")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_express_page_sidebar(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/plotly/test_plotly_app.py

@@ -1,11 +1,18 @@
+import pytest
 from playwright.sync_api import Page, expect
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 TIMEOUT = 2 * 60 * 1000
 app_url = create_deploys_app_url_fixture("example_deploy_app_A")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_deploys(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/deploys/shiny-client-console-error/test_shiny_client_error.py

@@ -1,10 +1,17 @@
+import pytest
 from playwright.sync_api import Page, expect
-from utils.deploy_utils import create_deploys_app_url_fixture, skip_if_not_chrome
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    reruns,
+    reruns_delay,
+    skip_if_not_chrome,
+)
 
 app_url = create_deploys_app_url_fixture("shiny_client_console_error")
 
 
 @skip_if_not_chrome
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 def test_shiny_client_console_error(page: Page, app_url: str) -> None:
     page.goto(app_url)
 

tests/playwright/utils/deploy_utils.py

@@ -4,6 +4,7 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 import time
 from typing import Any, Callable, TypeVar
@@ -12,6 +13,10 @@
 import requests
 from conftest import ScopeName, local_app_fixture_gen
 
+is_interactive = hasattr(sys, "ps1")
+reruns = 1 if is_interactive else 3
+reruns_delay = 1
+
 LOCAL_LOCATION = "local"
 
 __all__ = (