Arm backend: Fix sigmoid int16 and int32 flakyness

MLBEDSW-10642 has now been resolved. Step vela pin and remove all
related pytest.mark_flaky in sigmoid int16 and int32 tests.

Signed-off-by: Oscar Andersson <[email protected]>
Change-Id: I8c1aff31fb173fa6fdebc1d5b757623f9c2e1321

Author: oscarandersson8218

backends/arm/operator_support/tosa_supported_operators.py

@@ -317,6 +317,7 @@ class CheckProperQuantization(OperatorSupportBase):
         exir_ops.edge.aten.sub.Tensor,
         exir_ops.edge.aten.upsample_bilinear2d.vec,
         exir_ops.edge.aten.upsample_nearest2d.vec,
+        torch.ops.aten.scalar_tensor.default,
         *TableOps.included_ops(),
     )
 

backends/arm/test/ops/test_depthwise_conv.py

@@ -202,13 +202,13 @@ def test_convolution_2d_tosa_BI_depth_wise(test_module: torch.nn.Module):
 
 
 x_fails = {
-    "3x3_2x8x198x198_gp8_st3": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
-    "two_dw_conv2d": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
+    "3x3_2x8x198x198_gp8_st3": "MLETORCH-517: Operators fail with batches > 1",
+    "two_dw_conv2d": "MLETORCH-517: Operators fail with batches > 1",
 }
 
 
-@common.parametrize("test_module", testsuite_conv2d, x_fails)
 @common.XfailIfNoCorstone300  # TODO: MLETORCH-516
+@common.parametrize("test_module", testsuite_conv2d, x_fails)
 def test_convolution_2d_u55_BI_depth_wise(test_module: torch.nn.Module):
     pipeline = EthosU55PipelineBI[input_t](
         test_module(),
@@ -233,8 +233,8 @@ def test_convolution_1d_u55_BI_depth_wise(test_module: torch.nn.Module):
     pipeline.run()
 
 
-@common.parametrize("test_module", testsuite_conv1d | testsuite_conv2d, x_fails)
 @common.XfailIfNoCorstone320  # TODO: MLETORCH-516
+@common.parametrize("test_module", testsuite_conv2d, x_fails)
 def test_convolution_2d_u85_BI_depth_wise(test_module: torch.nn.Module):
     pipeline = EthosU85PipelineBI[input_t](
         test_module(),
@@ -244,3 +244,16 @@ def test_convolution_2d_u85_BI_depth_wise(test_module: torch.nn.Module):
         run_on_fvp=True,
     )
     pipeline.run()
+
+
+@common.XfailIfNoCorstone320  # TODO: MLETORCH-516
+@common.parametrize("test_module", testsuite_conv1d, x_fails)
+def test_convolution_1d_u85_BI_depth_wise(test_module: torch.nn.Module):
+    pipeline = EthosU85PipelineBI[input_t](
+        test_module(),
+        test_module().get_inputs(),
+        aten_ops=[],
+        exir_ops=exir_op,
+        run_on_fvp=True,
+    )
+    pipeline.run()

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -4,7 +4,6 @@
 # LICENSE file in the root directory of this source tree.
 
 import pytest
-
 import torch
 from executorch.backends.arm.quantizer import (
     get_symmetric_quantization_config,
@@ -91,10 +90,13 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        Sigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer())
     pipeline.run()
@@ -108,13 +110,13 @@ def test_sigmoid_tosa_BI(test_data):
     },
     strict=False,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI_add_sigmoid(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.run()
 
@@ -131,7 +133,6 @@ def test_sigmoid_tosa_BI_add_sigmoid(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_u55_BI(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(),
@@ -148,7 +149,6 @@ def test_sigmoid_u55_BI(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_u55_BI_add_sigmoid(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -163,7 +163,6 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_u85_BI(test_data):
     pipeline = EthosU85PipelineBI(
@@ -181,10 +180,10 @@ def test_sigmoid_u85_BI(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 @common.XfailIfNoCorstone320
 def test_sigmoid_u85_BI_add_sigmoid(test_data):
     pipeline = EthosU85PipelineBI(

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,7 +3,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import pytest
 import torch
 from executorch.backends.arm.quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -107,33 +106,32 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI_add_sigmoid(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_u55_BI(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(),
@@ -147,7 +145,6 @@ def test_sigmoid_u55_BI(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_u55_BI_add_sigmoid(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -162,9 +159,7 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
-@pytest.mark.flaky(reruns=5)
 def test_sigmoid_u85_BI(test_data):
     pipeline = EthosU85PipelineBI(
         Sigmoid(),
@@ -180,15 +175,8 @@ def test_sigmoid_u85_BI(test_data):
 @common.parametrize(
     "test_data",
     test_data_suite,
-    xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
-        "rand": "AssertionError: Output 0 does not match reference output.",
-        "rand_4d": "AssertionError: Output 0 does not match reference output.",
-    },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
-@pytest.mark.flaky(reruns=5)
 def test_sigmoid_u85_BI_add_sigmoid(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),

examples/arm/setup.sh

@@ -60,7 +60,7 @@ fi
 
 # vela
 vela_repo_url="https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela"
-vela_rev="859cc066178a87ff28230c1ce9bd370f1e98aa5a"
+vela_rev="8cac2b9a7204b57125a8718049519b091a98846c"
 
 ########
 ### Functions