[fuchsia][a11y] Adds inspect data to the a11y bridge. (#25868)

This change adds inspect data to the a11y bridge, which can be requested
via the command line.

Inspect nodes are lazy computed, meaning that they are only processed
when invoked, so no extra space is used during normal use.

Bug: fxbug.dev/75100

Test: AccessibilityBridgeTest.InspectData

Author: lucasradaelli

shell/platform/fuchsia/flutter/accessibility_bridge.cc

@@ -14,6 +14,7 @@
 #include <fuchsia/ui/gfx/cpp/fidl.h>
 #include <lib/fidl/cpp/binding_set.h>
 #include <lib/sys/cpp/service_directory.h>
+#include <lib/sys/inspect/cpp/component.h>
 #include <zircon/types.h>
 
 #include <memory>
@@ -40,13 +41,9 @@ namespace flutter_runner {
 class AccessibilityBridge
     : public fuchsia::accessibility::semantics::SemanticListener {
  public:
-  // A delegate to call when semantics are enabled or disabled.
-  class Delegate {
-   public:
-    virtual void SetSemanticsEnabled(bool enabled) = 0;
-    virtual void DispatchSemanticsAction(int32_t node_id,
-                                         flutter::SemanticsAction action) = 0;
-  };
+  using SetSemanticsEnabledCallback = std::function<void(bool)>;
+  using DispatchSemanticsActionCallback =
+      std::function<void(int32_t, flutter::SemanticsAction)>;
 
   // TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about
   // batching messages at this level.
@@ -67,9 +64,18 @@ class AccessibilityBridge
       "flutter::SemanticsNode::id and "
       "fuchsia::accessibility::semantics::Node::node_id differ in size.");
 
-  AccessibilityBridge(Delegate& delegate,
-                      const std::shared_ptr<sys::ServiceDirectory> services,
-                      fuchsia::ui::views::ViewRef view_ref);
+  AccessibilityBridge(
+      SetSemanticsEnabledCallback set_semantics_enabled_callback,
+      DispatchSemanticsActionCallback dispatch_semantics_action_callback,
+      const std::shared_ptr<sys::ServiceDirectory> services,
+      fuchsia::ui::views::ViewRef view_ref);
+
+  AccessibilityBridge(
+      SetSemanticsEnabledCallback set_semantics_enabled_callback,
+      DispatchSemanticsActionCallback dispatch_semantics_action_callback,
+      const std::shared_ptr<sys::ServiceDirectory> services,
+      inspect::Node inspect_node,
+      fuchsia::ui::views::ViewRef view_ref);
 
   // Returns true if accessible navigation is enabled.
   bool GetSemanticsEnabled() const;
@@ -105,28 +111,16 @@ class AccessibilityBridge
           OnAccessibilityActionRequestedCallback callback) override;
 
  private:
-  // Holds only the fields we need for hit testing.
+  // Fuchsia's default root semantic node id.
+  static constexpr int32_t kRootNodeId = 0;
+
+  // Holds a flutter semantic node and some extra info.
   // In particular, it adds a screen_rect field to flutter::SemanticsNode.
   struct SemanticsNode {
-    int32_t id;
-    int32_t flags;
-    bool is_focusable;
-    SkRect rect;
+    flutter::SemanticsNode data;
     SkRect screen_rect;
-    SkM44 transform;
-    std::vector<int32_t> children_in_hit_test_order;
   };
 
-  AccessibilityBridge::Delegate& delegate_;
-
-  static constexpr int32_t kRootNodeId = 0;
-  flutter::SemanticsNode root_flutter_semantics_node_;
-  float last_seen_view_pixel_ratio_ = 1.f;
-  fidl::Binding<fuchsia::accessibility::semantics::SemanticListener> binding_;
-  fuchsia::accessibility::semantics::SemanticsManagerPtr
-      fuchsia_semantics_manager_;
-  fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
-  bool semantics_enabled_;
   // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
   // Assists with pruning unreachable nodes and hit testing.
   std::unordered_map<int32_t, SemanticsNode> nodes_;
@@ -214,6 +208,34 @@ class AccessibilityBridge
   void OnSemanticsModeChanged(bool enabled,
                               OnSemanticsModeChangedCallback callback) override;
 
+#if !FLUTTER_RELEASE
+  // Fills the inspect tree with debug information about the semantic tree.
+  void FillInspectTree(int32_t flutter_node_id,
+                       int32_t current_level,
+                       inspect::Node inspect_node,
+                       inspect::Inspector* inspector) const;
+#endif  // !FLUTTER_RELEASE
+
+  SetSemanticsEnabledCallback set_semantics_enabled_callback_;
+  DispatchSemanticsActionCallback dispatch_semantics_action_callback_;
+  flutter::SemanticsNode root_flutter_semantics_node_;
+  float last_seen_view_pixel_ratio_ = 1.f;
+  fidl::Binding<fuchsia::accessibility::semantics::SemanticListener> binding_;
+  fuchsia::accessibility::semantics::SemanticsManagerPtr
+      fuchsia_semantics_manager_;
+  fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
+  bool semantics_enabled_;
+
+  // Node to publish inspect data.
+  inspect::Node inspect_node_;
+
+#if !FLUTTER_RELEASE
+  // Inspect node to store a dump of the semantic tree. Note that this only gets
+  // computed if requested, so it does not use memory to store the dump unless
+  // an explicit request is made.
+  inspect::LazyNode inspect_node_tree_dump_;
+#endif  // !FLUTTER_RELEASE
+
   FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
 };
 }  // namespace flutter_runner

shell/platform/fuchsia/flutter/accessibility_bridge.h

@@ -7,8 +7,12 @@
 #include <gtest/gtest.h>
 #include <lib/async-loop/cpp/loop.h>
 #include <lib/async-loop/default.h>
+#include <lib/async/cpp/executor.h>
 #include <lib/fidl/cpp/binding_set.h>
 #include <lib/fidl/cpp/interface_request.h>
+#include <lib/inspect/cpp/hierarchy.h>
+#include <lib/inspect/cpp/inspector.h>
+#include <lib/inspect/cpp/reader.h>
 #include <lib/sys/cpp/testing/service_directory_provider.h>
 #include <zircon/types.h>
 
@@ -33,12 +37,11 @@ void ExpectNodeHasRole(
 
 }  // namespace
 
-class AccessibilityBridgeTestDelegate
-    : public flutter_runner::AccessibilityBridge::Delegate {
+class AccessibilityBridgeTestDelegate {
  public:
-  void SetSemanticsEnabled(bool enabled) override { enabled_ = enabled; }
+  void SetSemanticsEnabled(bool enabled) { enabled_ = enabled; }
   void DispatchSemanticsAction(int32_t node_id,
-                               flutter::SemanticsAction action) override {
+                               flutter::SemanticsAction action) {
     actions.push_back(std::make_pair(node_id, action));
   }
 
@@ -53,7 +56,8 @@ class AccessibilityBridgeTest : public testing::Test {
  public:
   AccessibilityBridgeTest()
       : loop_(&kAsyncLoopConfigAttachToCurrentThread),
-        services_provider_(loop_.dispatcher()) {
+        services_provider_(loop_.dispatcher()),
+        executor_(loop_.dispatcher()) {
     services_provider_.AddService(
         semantics_manager_.GetHandler(loop_.dispatcher()),
         SemanticsManager::Name_);
@@ -64,16 +68,44 @@ class AccessibilityBridgeTest : public testing::Test {
     loop_.ResetQuit();
   }
 
+  void RunPromiseToCompletion(fit::promise<> promise) {
+    bool done = false;
+    executor_.schedule_task(
+        std::move(promise).and_then([&done]() { done = true; }));
+    while (loop_.GetState() == ASYNC_LOOP_RUNNABLE) {
+      if (done) {
+        loop_.ResetQuit();
+        return;
+      }
+
+      loop_.Run(zx::deadline_after(zx::duration::infinite()), true);
+    }
+    loop_.ResetQuit();
+  }
+
  protected:
   void SetUp() override {
     zx_status_t status = zx::eventpair::create(
         /*flags*/ 0u, &view_ref_control_.reference, &view_ref_.reference);
     EXPECT_EQ(status, ZX_OK);
 
     accessibility_delegate_.actions.clear();
+    inspector_ = std::make_unique<inspect::Inspector>();
+    flutter_runner::AccessibilityBridge::SetSemanticsEnabledCallback
+        set_semantics_enabled_callback = [this](bool enabled) {
+          accessibility_delegate_.SetSemanticsEnabled(enabled);
+        };
+    flutter_runner::AccessibilityBridge::DispatchSemanticsActionCallback
+        dispatch_semantics_action_callback =
+            [this](int32_t node_id, flutter::SemanticsAction action) {
+              accessibility_delegate_.DispatchSemanticsAction(node_id, action);
+            };
     accessibility_bridge_ =
         std::make_unique<flutter_runner::AccessibilityBridge>(
-            accessibility_delegate_, services_provider_.service_directory(),
+            std::move(set_semantics_enabled_callback),
+            std::move(dispatch_semantics_action_callback),
+            services_provider_.service_directory(),
+            inspector_->GetRoot().CreateChild("test_node"),
             std::move(view_ref_));
     RunLoopUntilIdle();
   }
@@ -85,10 +117,14 @@ class AccessibilityBridgeTest : public testing::Test {
   MockSemanticsManager semantics_manager_;
   AccessibilityBridgeTestDelegate accessibility_delegate_;
   std::unique_ptr<flutter_runner::AccessibilityBridge> accessibility_bridge_;
+  // Required to verify inspect metrics.
+  std::unique_ptr<inspect::Inspector> inspector_;
 
  private:
   async::Loop loop_;
   sys::testing::ServiceDirectoryProvider services_provider_;
+  // Required to retrieve inspect metrics.
+  async::Executor executor_;
 };
 
 TEST_F(AccessibilityBridgeTest, RegistersViewRef) {
@@ -952,4 +988,54 @@ TEST_F(AccessibilityBridgeTest, Actions) {
   EXPECT_EQ(accessibility_delegate_.actions.back(),
             std::make_pair(0, flutter::SemanticsAction::kDecrease));
 }
+
+#if !FLUTTER_RELEASE
+TEST_F(AccessibilityBridgeTest, InspectData) {
+  flutter::SemanticsNodeUpdates updates;
+  flutter::SemanticsNode node0;
+  node0.id = 0;
+  node0.label = "node0";
+  node0.hint = "node0_hint";
+  node0.value = "value";
+  node0.flags |= static_cast<int>(flutter::SemanticsFlags::kIsButton);
+  node0.childrenInTraversalOrder = {1};
+  node0.childrenInHitTestOrder = {1};
+  node0.rect.setLTRB(0, 0, 100, 100);
+  updates.emplace(0, node0);
+
+  flutter::SemanticsNode node1;
+  node1.id = 1;
+  node1.flags |= static_cast<int>(flutter::SemanticsFlags::kIsHeader);
+  node1.childrenInTraversalOrder = {};
+  node1.childrenInHitTestOrder = {};
+  updates.emplace(1, node1);
+
+  accessibility_bridge_->AddSemanticsNodeUpdate(std::move(updates), 1.f);
+  RunLoopUntilIdle();
+
+  fit::result<inspect::Hierarchy> hierarchy;
+  ASSERT_FALSE(hierarchy.is_ok());
+  RunPromiseToCompletion(
+      inspect::ReadFromInspector(*inspector_)
+          .then([&hierarchy](fit::result<inspect::Hierarchy>& result) {
+            hierarchy = std::move(result);
+          }));
+  ASSERT_TRUE(hierarchy.is_ok());
+
+  auto tree_inspect_hierarchy = hierarchy.value().GetByPath({"test_node"});
+  ASSERT_NE(tree_inspect_hierarchy, nullptr);
+  // TODO(http://fxbug.dev/75841): Rewrite flutter engine accessibility bridge
+  // tests using inspect matchers. The checks bellow verify that the tree was
+  // built, and that it matches the format of the input tree. This will be
+  // updated in the future when test matchers are available to verify individual
+  // property values.
+  const auto& root = tree_inspect_hierarchy->children();
+  ASSERT_EQ(root.size(), 1u);
+  EXPECT_EQ(root[0].name(), "semantic_tree_root");
+  const auto& child = root[0].children();
+  ASSERT_EQ(child.size(), 1u);
+  EXPECT_EQ(child[0].name(), "node_1");
+}
+#endif  // !FLUTTER_RELEASE
+
 }  // namespace flutter_runner_test

shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc

@@ -102,6 +102,8 @@ Engine::Engine(Delegate& delegate,
   // refs are not copyable, and multiple consumers need view refs.
   fuchsia::ui::views::ViewRef platform_view_ref;
   view_ref_pair.view_ref.Clone(&platform_view_ref);
+  fuchsia::ui::views::ViewRef accessibility_bridge_view_ref;
+  view_ref_pair.view_ref.Clone(&accessibility_bridge_view_ref);
   fuchsia::ui::views::ViewRef isolate_view_ref;
   view_ref_pair.view_ref.Clone(&isolate_view_ref);
   // Input3 keyboard listener registration requires a ViewRef as an event
@@ -171,6 +173,30 @@ Engine::Engine(Delegate& delegate,
   environment->GetServices(parent_environment_service_provider.NewRequest());
   environment.Unbind();
 
+  AccessibilityBridge::SetSemanticsEnabledCallback
+      set_semantics_enabled_callback = [this](bool enabled) {
+        auto platform_view = shell_->GetPlatformView();
+
+        if (platform_view) {
+          platform_view->SetSemanticsEnabled(enabled);
+        }
+      };
+
+  AccessibilityBridge::DispatchSemanticsActionCallback
+      dispatch_semantics_action_callback =
+          [this](int32_t node_id, flutter::SemanticsAction action) {
+            auto platform_view = shell_->GetPlatformView();
+
+            if (platform_view) {
+              platform_view->DispatchSemanticsAction(node_id, action, {});
+            }
+          };
+
+  accessibility_bridge_ = std::make_unique<AccessibilityBridge>(
+      std::move(set_semantics_enabled_callback),
+      std::move(dispatch_semantics_action_callback), svc,
+      std::move(accessibility_bridge_view_ref));
+
   OnEnableWireframe on_enable_wireframe_callback = std::bind(
       &Engine::DebugWireframeSettingsChanged, this, std::placeholders::_1);
 
@@ -227,6 +253,16 @@ Engine::Engine(Delegate& delegate,
   keyboard_svc_->AddListener(std::move(keyboard_view_ref),
                              keyboard_listener.Bind(), [] {});
 
+  OnSemanticsNodeUpdate on_semantics_node_update_callback =
+      [this](flutter::SemanticsNodeUpdates updates, float pixel_ratio) {
+        accessibility_bridge_->AddSemanticsNodeUpdate(updates, pixel_ratio);
+      };
+
+  OnRequestAnnounce on_request_announce_callback =
+      [this](const std::string& message) {
+        accessibility_bridge_->RequestAnnounce(message);
+      };
+
   // Setup the callback that will instantiate the platform view.
   flutter::Shell::CreateCallback<flutter::PlatformView>
       on_create_platform_view = fml::MakeCopyable(
@@ -244,6 +280,10 @@ Engine::Engine(Delegate& delegate,
            on_update_view_callback = std::move(on_update_view_callback),
            on_destroy_view_callback = std::move(on_destroy_view_callback),
            on_create_surface_callback = std::move(on_create_surface_callback),
+           on_semantics_node_update_callback =
+               std::move(on_semantics_node_update_callback),
+           on_request_announce_callback =
+               std::move(on_request_announce_callback),
            external_view_embedder = GetExternalViewEmbedder(),
            keyboard_listener_request = std::move(keyboard_listener_request),
            await_vsync_callback =
@@ -271,7 +311,9 @@ Engine::Engine(Delegate& delegate,
                 std::move(on_create_view_callback),
                 std::move(on_update_view_callback),
                 std::move(on_destroy_view_callback),
-                std::move(on_create_surface_callback), external_view_embedder,
+                std::move(on_create_surface_callback),
+                std::move(on_semantics_node_update_callback),
+                std::move(on_request_announce_callback), external_view_embedder,
                 // Callbacks for VsyncWaiter to call into SessionConnection.
                 await_vsync_callback,
                 await_vsync_for_secondary_callback_callback);

shell/platform/fuchsia/flutter/engine.cc

@@ -21,6 +21,7 @@
 #include "flutter/flow/surface.h"
 #include "flutter/fml/macros.h"
 #include "flutter/shell/common/shell.h"
+#include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h"
 
 #include "default_session_connection.h"
 #include "flutter_runner_product_configuration.h"
@@ -83,6 +84,7 @@ class Engine final {
 
   std::unique_ptr<IsolateConfigurator> isolate_configurator_;
   std::unique_ptr<flutter::Shell> shell_;
+  std::unique_ptr<AccessibilityBridge> accessibility_bridge_;
 
   fuchsia::intl::PropertyProviderPtr intl_property_provider_;