From edc25b1ede49b7c734a3aff20c75907f5e016367 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 24 Oct 2022 10:53:34 -0400 Subject: [PATCH 1/6] Add a shard for vulkan testing --- ci/builders/linux_host_engine.json | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ci/builders/linux_host_engine.json b/ci/builders/linux_host_engine.json index 1c1f621464363..461b9671690ba 100644 --- a/ci/builders/linux_host_engine.json +++ b/ci/builders/linux_host_engine.json @@ -26,6 +26,48 @@ "tests": [] }, { + "archives": [], + "drone_dimensions": [ + "device_type=none", + "os=Linux" + ], + "gclient_custom_vars": { + "download_android_deps": false + }, + "gn": [ + "--runtime-mode", + "debug", + "--unoptimized", + "--prebuilt-dart-sdk", + "--enable-impeller-vulkan", + "--target-dir", + "host_debug_impeller_vulkan" + ], + "name": "host_debug_impeller_vulkan", + "ninja": { + "config": "host_debug_impeller_vulkan", + "targets": [ + "flutter", + "flutter/sky/packages" + ] + }, + "tests": [ + { + "language": "python", + "name": "Host Tests for host_debug_impeller_vulkan", + "parameters": [ + "--variant", + "host_debug_impeller_vulkan", + "--type", + "engine", + "--engine-capture-core-dump" + ], + "script": "flutter/testing/run_tests.py", + "type": "local" + } + ] + }, + { "archives": [ { "name": "host_debug", From 3244a322455599d489c3f267c54d8fa61e1839cc Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 24 Oct 2022 14:57:48 -0400 Subject: [PATCH 2/6] Filter to passing vulkan tests --- ci/builders/linux_host_engine.json | 2 +- testing/impeller_vulkan_test_status.csv | 141 ++++++++++++++++++++++++ testing/run_tests.py | 41 ++++++- 3 files changed, 178 insertions(+), 6 deletions(-) create mode 100644 testing/impeller_vulkan_test_status.csv diff --git a/ci/builders/linux_host_engine.json b/ci/builders/linux_host_engine.json index 461b9671690ba..876fca52c101c 100644 --- a/ci/builders/linux_host_engine.json +++ b/ci/builders/linux_host_engine.json @@ -59,7 +59,7 @@ "--variant", "host_debug_impeller_vulkan", "--type", - "engine", + "impeller-vulkan", "--engine-capture-core-dump" ], "script": "flutter/testing/run_tests.py", diff --git a/testing/impeller_vulkan_test_status.csv b/testing/impeller_vulkan_test_status.csv new file mode 100644 index 0000000000000..34ff0db0bec99 --- /dev/null +++ b/testing/impeller_vulkan_test_status.csv @@ -0,0 +1,141 @@ +test,status +CanRenderImage/Vulkan,fail +CanRenderTiledTexture/Vulkan,fail +CanRenderImageRect/Vulkan,fail +CoordinateConversionsAreCorrect/Vulkan,fail +CanRenderDifferencePaths/Vulkan,fail +SaveLayerFiltersScaleWithTransform/Vulkan,fail +CanDrawImage/Vulkan,fail +CanDrawWithMaskBlur/Vulkan,fail +CanDrawWithBlendColorFilter/Vulkan,fail +CanDrawWithColorFilterImageFilter/Vulkan,fail +CanDrawWithImageBlurFilter/Vulkan,fail +CanDrawWithComposeImageFilter/Vulkan,fail +CanClampTheResultingColorOfColorMatrixFilter/Vulkan,fail +SaveLayerWithColorMatrixFiltersAndAlphaDrawCorrectly/Vulkan,fail +SaveLayerWithBlendFiltersAndAlphaDrawCorrectly/Vulkan,fail +CanDrawBackdropFilter/Vulkan,fail +CanDrawNinePatchImage/Vulkan,fail +CanDrawNinePatchImageCenterWidthBiggerThanDest/Vulkan,fail +CanDrawNinePatchImageCenterHeightBiggerThanDest/Vulkan,fail +CanDrawNinePatchImageCenterBiggerThanDest/Vulkan,fail +CanDrawNinePatchImageCornersScaledDown/Vulkan,fail +CanDrawWithMatrixFilter/Vulkan,fail +CanDrawPaintWithColorSource/Vulkan,fail +FilterCoverageRespectsCropRect/Vulkan,fail +Filters/Vulkan,fail +GaussianBlurFilter/Vulkan,fail +MorphologyFilter/Vulkan,fail +DrawAtlasNoColor/Vulkan,fail +DrawAtlasWithColor/Vulkan,fail +DrawAtlasUsesProvidedCullRectForCoverage/Vulkan,fail +DrawAtlasWithOpacity/Vulkan,fail +DrawAtlasNoColorFullSize/Vulkan,fail +ColorMatrixFilterEditable/Vulkan,fail +LinearToSrgbFilter/Vulkan,fail +SrgbToLinearFilter/Vulkan,fail +CanCreateBoxPrimitive/Vulkan,fail +CanRenderPerspectiveCube/Vulkan,fail +CanRenderMultiplePrimitives/Vulkan,fail +CanRenderToTexture/Vulkan,fail +CanBlitTextureToTexture/Vulkan,fail +CanGenerateMipmaps/Vulkan,fail +TheImpeller/Vulkan,fail +InactiveUniforms/Vulkan,fail +CanCreateGlyphAtlas/Vulkan,fail +GlyphAtlasWithOddUniqueGlyphSize/Vulkan,fail +CanRegisterStage/Vulkan,pass +CanCreatePipelineFromRuntimeStage/Vulkan,pass +CanvasCTMCanBeUpdated/Vulkan,pass +CanvasCanPushPopCTM/Vulkan,pass +CanRenderColoredRect/Vulkan,pass +CanRenderStrokes/Vulkan,pass +CanRenderCurvedStrokes/Vulkan,pass +CanRenderClips/Vulkan,pass +CanRenderNestedClips/Vulkan,pass +CanRenderDifferenceClips/Vulkan,pass +ClipsUseCurrentTransform/Vulkan,pass +CanSaveLayerStandalone/Vulkan,pass +CanRenderLinearGradient/Vulkan,pass +CanRenderLinearGradientManyColors/Vulkan,pass +CanRenderLinearGradientWayManyColors/Vulkan,pass +CanRenderLinearGradientManyColorsUnevenStops/Vulkan,pass +CanRenderRadialGradient/Vulkan,pass +CanRenderRadialGradientManyColors/Vulkan,pass +CanRenderSweepGradient/Vulkan,pass +CanRenderSweepGradientManyColors/Vulkan,pass +CanRenderDifferentShapesWithSameColorSource/Vulkan,pass +CanPictureConvertToImage/Vulkan,pass +BlendModeShouldCoverWholeScreen/Vulkan,pass +CanRenderGroupOpacity/Vulkan,pass +CanPerformFullScreenMSAA/Vulkan,pass +CanPerformSkew/Vulkan,pass +CanPerformSaveLayerWithBounds/Vulkan,pass +CanPerformSaveLayerWithBoundsAndLargerIntermediateIsNotAllocated/Vulkan,pass +CanRenderRoundedRectWithNonUniformRadii/Vulkan,pass +CanRenderTextFrame/Vulkan,pass +CanRenderItalicizedText/Vulkan,pass +CanRenderEmojiTextFrame/Vulkan,pass +CanRenderTextInSaveLayer/Vulkan,pass +CanDrawPaint/Vulkan,pass +PaintBlendModeIsRespected/Vulkan,pass +ColorWheel/Vulkan,pass +TransformMultipliesCorrectly/Vulkan,pass +SolidStrokesRenderCorrectly/Vulkan,pass +GradientStrokesRenderCorrectly/Vulkan,pass +CoverageOriginShouldBeAccountedForInSubpasses/Vulkan,pass +DrawRectStrokesRenderCorrectly/Vulkan,pass +SaveLayerDrawsBehindSubsequentEntities/Vulkan,pass +SiblingSaveLayerBoundsAreRespected/Vulkan,pass +CanRenderClippedLayers/Vulkan,pass +CanDrawRect/Vulkan,pass +CanDrawTextBlob/Vulkan,pass +CanDrawCapsAndJoins/Vulkan,pass +CanDrawArc/Vulkan,pass +StrokedPathsDrawCorrectly/Vulkan,pass +CanDrawWithOddPathWinding/Vulkan,pass +CanDrawPoints/Vulkan,pass +CanDrawZeroLengthLine/Vulkan,pass +CanDrawShadow/Vulkan,pass +CanConvertTriangleFanToTriangles/Vulkan,pass +CanDrawZeroWidthLine/Vulkan,pass +CanDrawRectWithLinearToSrgbColorFilter/Vulkan,pass +CanBlendDstOverAndDstCorrectly/Vulkan,pass +CanCreateEntity/Vulkan,pass +EntityPassCoverageRespectsDelegateBoundsHint/Vulkan,pass +EntityPassCoverageRespectsCoverageLimit/Vulkan,pass +CanDrawRect/Vulkan,pass +ThreeStrokesInOnePath/Vulkan,pass +TriangleInsideASquare/Vulkan,pass +StrokeCapAndJoinTest/Vulkan,pass +CubicCurveTest/Vulkan,pass +CubicCurveAndOverlapTest/Vulkan,pass +SolidColorContentsStrokeSetStrokeCapsAndJoins/Vulkan,pass +SolidColorContentsStrokeSetMiterLimit/Vulkan,pass +BlendingModeOptions/Vulkan,pass +BezierCircleScaled/Vulkan,pass +SetBlendMode/Vulkan,pass +ContentsGetBoundsForEmptyPathReturnsNullopt/Vulkan,pass +SolidStrokeCoverageIsCorrect/Vulkan,pass +BorderMaskBlurCoverageIsCorrect/Vulkan,pass +DrawVerticesSolidColorTrianglesWithoutIndices/Vulkan,pass +DrawVerticesLinearGradientWithoutIndices/Vulkan,pass +DrawVerticesSolidColorTrianglesWithIndices/Vulkan,pass +SolidFillCoverageIsCorrect/Vulkan,pass +SolidFillShouldRenderIsCorrect/Vulkan,pass +ClipContentsShouldRenderIsCorrect/Vulkan,pass +RRectShadowTest/Vulkan,pass +ColorMatrixFilterCoverageIsCorrect/Vulkan,pass +LinearToSrgbFilterCoverageIsCorrect/Vulkan,pass +SrgbToLinearFilterCoverageIsCorrect/Vulkan,pass +TTTBlendColor/Vulkan,pass +SdfText/Vulkan,pass +RuntimeEffect/Vulkan,pass +ArrayUniforms/Vulkan,pass +CanCreateCPUBackedTexture/Vulkan,pass +DefaultIndexSize/Vulkan,pass +VertexBufferBuilder/Vulkan,pass +CanCreateComputePass/Vulkan,pass +CanConvertTextBlob/Vulkan,pass +CanCreateRenderContext/Vulkan,pass +LazyAtlasTracksColor/Vulkan,pass diff --git a/testing/run_tests.py b/testing/run_tests.py index 66799a5be9aa4..d8d2acd27c0c6 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -17,7 +17,9 @@ import subprocess import sys import time +import csv +script_dir = os.path.dirname(os.path.realpath(__file__)) buildroot_dir = os.path.abspath( os.path.join(os.path.realpath(__file__), '..', '..', '..') ) @@ -305,6 +307,12 @@ def __str__(self): return " ".join(command) +shuffle_flags = [ + "--gtest_repeat=2", + "--gtest_shuffle", +] + + def RunCCTests(build_dir, filter, coverage, capture_core_dump): print("Running Engine Unit-tests.") @@ -314,11 +322,6 @@ def RunCCTests(build_dir, filter, coverage, capture_core_dump): resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY) ) - shuffle_flags = [ - "--gtest_repeat=2", - "--gtest_shuffle", - ] - repeat_flags = [ "--repeat=2", ] @@ -427,6 +430,19 @@ def make_test(name, flags=repeat_flags, extra_env={}): ) +def ParseImpellerVulkanFilter(): + test_status_path = os.path.join(script_dir, 'impeller_vulkan_test_status.csv') + gtest_filter = '--gtest_filter="' + with open(test_status_path, 'r') as csvfile: + csvreader = csv.reader(csvfile) + next(csvreader) # Skip header. + for row in csvreader: + if row[1] == 'pass': + gtest_filter += '*%s:' % row[0] + gtest_filter += '"' + return gtest_filter + + def RunEngineBenchmarks(build_dir, filter): print("Running Engine Benchmarks.") @@ -1054,6 +1070,21 @@ def main(): build_dir, engine_filter, args.coverage, args.engine_capture_core_dump ) + # Use this type to exclusively run impeller vulkan tests. + # TODO (https://github.com/flutter/flutter/issues/113961): Remove this once + # impeller vulkan tests are stable. + if 'impeller-vulkan' in types: + vulkan_gtest_filter = ParseImpellerVulkanFilter() + gtest_flags = shuffle_flags + gtest_flags.append(vulkan_gtest_filter) + RunEngineExecutable( + build_dir, + 'impeller_unittests', + engine_filter, + gtest_flags, + coverage=args.coverage + ) + if 'dart' in types: dart_filter = args.dart_filter.split(',') if args.dart_filter else None tasks = list(GatherDartSmokeTest(build_dir, args.verbose_dart_snapshot)) From a6e41d87155a42ef7236e554a963dcf53fa7cd2e Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 24 Oct 2022 16:49:48 -0400 Subject: [PATCH 3/6] set error callback before init --- impeller/playground/playground.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impeller/playground/playground.cc b/impeller/playground/playground.cc index adfce380cc838..312713c264385 100644 --- a/impeller/playground/playground.cc +++ b/impeller/playground/playground.cc @@ -62,11 +62,11 @@ struct Playground::GLFWInitializer { // applicationDidFinishLaunching is never fired. static std::once_flag sOnceInitializer; std::call_once(sOnceInitializer, []() { - FML_CHECK(::glfwInit() == GLFW_TRUE); ::glfwSetErrorCallback([](int code, const char* description) { FML_LOG(ERROR) << "GLFW Error '" << description << "' (" << code << ")."; }); + FML_CHECK(::glfwInit() == GLFW_TRUE); }); } }; From 71fcb5b277691f84f3c197f88b253b041b6b8192 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 24 Oct 2022 17:10:40 -0400 Subject: [PATCH 4/6] use a hidden window --- impeller/playground/backend/vulkan/playground_impl_vk.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/impeller/playground/backend/vulkan/playground_impl_vk.cc b/impeller/playground/backend/vulkan/playground_impl_vk.cc index 334d35e1d6289..bcd35f85ecb3b 100644 --- a/impeller/playground/backend/vulkan/playground_impl_vk.cc +++ b/impeller/playground/backend/vulkan/playground_impl_vk.cc @@ -54,6 +54,7 @@ PlaygroundImplVK::PlaygroundImplVK() ::glfwDefaultWindowHints(); ::glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); ::glfwWindowHint(GLFW_RESIZABLE, GLFW_FALSE); + ::glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); auto window = ::glfwCreateWindow(800, 600, "Test Vulkan Window", nullptr, nullptr); From 829d7343b58f6b5ea3aed3c447e20135338d2d2c Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Tue, 25 Oct 2022 13:19:58 -0400 Subject: [PATCH 5/6] use xvfb --- testing/run_tests.py | 4 ++ testing/xvfb.py | 154 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 testing/xvfb.py diff --git a/testing/run_tests.py b/testing/run_tests.py index d8d2acd27c0c6..babdd78443678 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -18,6 +18,7 @@ import sys import time import csv +import xvfb script_dir = os.path.dirname(os.path.realpath(__file__)) buildroot_dir = os.path.abspath( @@ -1074,6 +1075,8 @@ def main(): # TODO (https://github.com/flutter/flutter/issues/113961): Remove this once # impeller vulkan tests are stable. if 'impeller-vulkan' in types: + build_name = args.variant + xvfb.StartVirtualX(build_name, build_dir) vulkan_gtest_filter = ParseImpellerVulkanFilter() gtest_flags = shuffle_flags gtest_flags.append(vulkan_gtest_filter) @@ -1084,6 +1087,7 @@ def main(): gtest_flags, coverage=args.coverage ) + xvfb.StopVirtualX(build_name) if 'dart' in types: dart_filter = args.dart_filter.split(',') if args.dart_filter else None diff --git a/testing/xvfb.py b/testing/xvfb.py new file mode 100644 index 0000000000000..b7f6f40411d8b --- /dev/null +++ b/testing/xvfb.py @@ -0,0 +1,154 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Functions to setup xvfb, which is used by the linux machines. +""" + +import os +import platform +import signal +import subprocess +import tempfile +import time + + +def _XvfbDisplayIndex(_child_build_name): + return '9' + + +def _XvfbPidFilename(child_build_name): + """Returns the filename to the Xvfb pid file. This name is unique for each + builder. This is used by the linux builders.""" + return os.path.join( + tempfile.gettempdir(), + 'xvfb-' + _XvfbDisplayIndex(child_build_name) + '.pid' + ) + + +def StartVirtualX(child_build_name, build_dir): + """Start a virtual X server and set the DISPLAY environment variable so sub + processes will use the virtual X server. Also start openbox. This only works + on Linux and assumes that xvfb and openbox are installed. + + Args: + child_build_name: The name of the build that we use for the pid file. + E.g., webkit-rel-linux. + build_dir: The directory where binaries are produced. If this is non-empty, + we try running xdisplaycheck from |build_dir| to verify our X + connection. + """ + # We use a pid file to make sure we don't have any xvfb processes running + # from a previous test run. + StopVirtualX(child_build_name) + + xdisplaycheck_path = None + if build_dir: + xdisplaycheck_path = os.path.join(build_dir, 'xdisplaycheck') + + display = ':%s' % _XvfbDisplayIndex(child_build_name) + # Note we don't add the optional screen here (+ '.0') + os.environ['DISPLAY'] = display + + # Parts of Xvfb use a hard-coded "/tmp" for its temporary directory. + # This can cause a failure when those parts expect to hardlink against + # files that were created in "TEMPDIR" / "TMPDIR". + # + # See: https://crbug.com/715848 + env = os.environ.copy() + if env.get('TMPDIR') and env['TMPDIR'] != '/tmp': + print('Overriding TMPDIR to "/tmp" for Xvfb, was: %s' % (env['TMPDIR'],)) + env['TMPDIR'] = '/tmp' + + if xdisplaycheck_path and os.path.exists(xdisplaycheck_path): + print('Verifying Xvfb is not running ...') + checkstarttime = time.time() + xdisplayproc = subprocess.Popen([xdisplaycheck_path, '--noserver'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env) + # Wait for xdisplaycheck to exit. + logs = xdisplayproc.communicate()[0] + if xdisplayproc.returncode == 0: + print('xdisplaycheck says there is a display still running, exiting...') + raise Exception('Display already present.') + + xvfb_lock_filename = '/tmp/.X%s-lock' % _XvfbDisplayIndex(child_build_name) + if os.path.exists(xvfb_lock_filename): + print('Removing stale xvfb lock file %r' % xvfb_lock_filename) + try: + os.unlink(xvfb_lock_filename) + except OSError as e: + print('Removing xvfb lock file failed: %s' % e) + + # Figure out which X server to try. + cmd = 'Xvfb' + + # Start a virtual X server that we run the tests in. This makes it so we can + # run the tests even if we didn't start the tests from an X session. + proc = subprocess.Popen([ + cmd, display, '-screen', '0', '1280x800x24', '-ac', '-dpi', '96', + '-maxclients', '512' + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env) + xvfb_pid_filename = _XvfbPidFilename(child_build_name) + open(xvfb_pid_filename, 'w').write(str(proc.pid)) + + # Wait for Xvfb to start up. + time.sleep(10) + + # Verify that Xvfb has started by using xdisplaycheck. + if xdisplaycheck_path and os.path.exists(xdisplaycheck_path): + print('Verifying Xvfb has started...') + checkstarttime = time.time() + xdisplayproc = subprocess.Popen([xdisplaycheck_path], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Wait for xdisplaycheck to exit. + logs = xdisplayproc.communicate()[0] + checktime = time.time() - checkstarttime + if xdisplayproc.returncode != 0: + print('xdisplaycheck failed after %d seconds.' % checktime) + print('xdisplaycheck output:') + for l in logs.splitlines(): + print('> %s' % l) + rc = proc.poll() + if rc is None: + print('Xvfb still running, stopping.') + proc.terminate() + else: + print('Xvfb exited, code %d' % rc) + + print('Xvfb output:') + for l in proc.communicate()[0].splitlines(): + print('> %s' % l) + raise Exception(logs) + else: + print('xdisplaycheck succeeded after %d seconds.' % checktime) + print('xdisplaycheck output:') + for l in logs.splitlines(): + print('> %s' % l) + print('...OK') + + # Some ChromeOS tests need a window manager. + subprocess.Popen('openbox', stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print('Window manager (openbox) started.') + + +def StopVirtualX(child_build_name): + """Try and stop the virtual X server if one was started with StartVirtualX. + When the X server dies, it takes down the window manager with it. + If a virtual x server is not running, this method does nothing.""" + xvfb_pid_filename = _XvfbPidFilename(child_build_name) + if os.path.exists(xvfb_pid_filename): + xvfb_pid = int(open(xvfb_pid_filename).read()) + print('Stopping Xvfb with pid %d ...' % xvfb_pid) + # If the process doesn't exist, we raise an exception that we can ignore. + try: + os.kill(xvfb_pid, signal.SIGKILL) + except OSError: + print('... killing failed, presuming unnecessary.') + os.remove(xvfb_pid_filename) + print('Xvfb pid file removed') From 729e8b3d551a4d47f2762c887c246b81e971d4d7 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Tue, 25 Oct 2022 14:34:32 -0400 Subject: [PATCH 6/6] Stop Xvfb in finally block --- testing/run_tests.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index babdd78443678..af11bea364f21 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -1076,18 +1076,20 @@ def main(): # impeller vulkan tests are stable. if 'impeller-vulkan' in types: build_name = args.variant - xvfb.StartVirtualX(build_name, build_dir) - vulkan_gtest_filter = ParseImpellerVulkanFilter() - gtest_flags = shuffle_flags - gtest_flags.append(vulkan_gtest_filter) - RunEngineExecutable( - build_dir, - 'impeller_unittests', - engine_filter, - gtest_flags, - coverage=args.coverage - ) - xvfb.StopVirtualX(build_name) + try: + xvfb.StartVirtualX(build_name, build_dir) + vulkan_gtest_filter = ParseImpellerVulkanFilter() + gtest_flags = shuffle_flags + gtest_flags.append(vulkan_gtest_filter) + RunEngineExecutable( + build_dir, + 'impeller_unittests', + engine_filter, + gtest_flags, + coverage=args.coverage + ) + finally: + xvfb.StopVirtualX(build_name) if 'dart' in types: dart_filter = args.dart_filter.split(',') if args.dart_filter else None