diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 7f37acc4cc8e2..b436d7548da3a 100755 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1198,6 +1198,9 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterSurfa FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm +FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h +FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.mm +FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObjectTest.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.h FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h diff --git a/shell/platform/common/accessibility_bridge.cc b/shell/platform/common/accessibility_bridge.cc index e68a18ae371ff..723492f3a64ea 100644 --- a/shell/platform/common/accessibility_bridge.cc +++ b/shell/platform/common/accessibility_bridge.cc @@ -111,6 +111,22 @@ AccessibilityBridge::GetPendingEvents() { return result; } +void AccessibilityBridge::UpdateDelegate( + std::unique_ptr delegate) { + delegate_ = std::move(delegate); + // Recreate FlutterPlatformNodeDelegates since they may contain stale state + // from the previous AccessibilityBridgeDelegate. + for (const auto& [node_id, old_platform_node_delegate] : id_wrapper_map_) { + std::shared_ptr platform_node_delegate = + delegate_->CreateFlutterPlatformNodeDelegate(); + platform_node_delegate->Init( + std::static_pointer_cast( + shared_from_this()), + old_platform_node_delegate->GetAXNode()); + id_wrapper_map_[node_id] = platform_node_delegate; + } +} + void AccessibilityBridge::OnNodeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) {} @@ -329,7 +345,7 @@ void AccessibilityBridge::SetBooleanAttributesFromFlutterUpdate( node_data.AddBoolAttribute( ax::mojom::BoolAttribute::kEditableRoot, flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField && - (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) > 0); + (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0); } void AccessibilityBridge::SetIntAttributesFromFlutterUpdate( diff --git a/shell/platform/common/accessibility_bridge.h b/shell/platform/common/accessibility_bridge.h index ab8969ab47cb7..7a2b0a6c95d69 100644 --- a/shell/platform/common/accessibility_bridge.h +++ b/shell/platform/common/accessibility_bridge.h @@ -97,7 +97,7 @@ class AccessibilityBridge /// by accessibility bridge whenever a new AXNode is created in /// AXTree. Each platform needs to implement this method in /// order to inject its subclass into the accessibility bridge. - virtual std::unique_ptr + virtual std::shared_ptr CreateFlutterPlatformNodeDelegate() = 0; }; //----------------------------------------------------------------------------- @@ -163,6 +163,11 @@ class AccessibilityBridge /// all pending events. const std::vector GetPendingEvents(); + //------------------------------------------------------------------------------ + /// @brief Update the AccessibilityBridgeDelegate stored in the + /// accessibility bridge to a new one. + void UpdateDelegate(std::unique_ptr delegate); + private: // See FlutterSemanticsNode in embedder.h typedef struct { diff --git a/shell/platform/common/accessibility_bridge_unittests.cc b/shell/platform/common/accessibility_bridge_unittests.cc index 90d50a571fcd5..c7119a0071dda 100644 --- a/shell/platform/common/accessibility_bridge_unittests.cc +++ b/shell/platform/common/accessibility_bridge_unittests.cc @@ -155,6 +155,66 @@ TEST(AccessibilityBridgeTest, canFireChildrenChangedCorrectly) { actual_event.end()); } +TEST(AccessibilityBridgeTest, canUpdateDelegate) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.flags = static_cast(0); + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.flags = static_cast(0); + child1.actions = static_cast(0); + child1.text_selection_base = -1; + child1.text_selection_extent = -1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + + auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1); + EXPECT_FALSE(root_node.expired()); + EXPECT_FALSE(child1_node.expired()); + // Update Delegate + bridge->UpdateDelegate(std::make_unique()); + + // Old tree is destroyed. + EXPECT_TRUE(root_node.expired()); + EXPECT_TRUE(child1_node.expired()); + + // New tree still has the data. + auto new_root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + auto new_child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + EXPECT_EQ(new_root_node->GetChildCount(), 1); + EXPECT_EQ(new_root_node->GetData().child_ids[0], 1); + EXPECT_EQ(new_root_node->GetName(), "root"); + + EXPECT_EQ(new_child1_node->GetChildCount(), 0); + EXPECT_EQ(new_child1_node->GetName(), "child 1"); +} + TEST(AccessibilityBridgeTest, canHandleSelectionChangeCorrectly) { TestAccessibilityBridgeDelegate* delegate = new TestAccessibilityBridgeDelegate(); @@ -200,5 +260,34 @@ TEST(AccessibilityBridgeTest, canHandleSelectionChangeCorrectly) { ui::AXEventGenerator::Event::OTHER_ATTRIBUTE_CHANGED); } +TEST(AccessibilityBridgeTest, doesNotAssignEditableRootToSelectableText) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.flags = static_cast( + FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField | + FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly); + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 0; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + bridge->CommitUpdates(); + + auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + + EXPECT_FALSE(root_node->GetData().GetBoolAttribute( + ax::mojom::BoolAttribute::kEditableRoot)); +} + } // namespace testing } // namespace flutter diff --git a/shell/platform/common/flutter_platform_node_delegate.cc b/shell/platform/common/flutter_platform_node_delegate.cc index 9eff09cc19350..8f636909a243d 100644 --- a/shell/platform/common/flutter_platform_node_delegate.cc +++ b/shell/platform/common/flutter_platform_node_delegate.cc @@ -107,4 +107,9 @@ gfx::Rect FlutterPlatformNodeDelegate::GetBoundsRect( return gfx::ToEnclosingRect(bounds); } +std::weak_ptr +FlutterPlatformNodeDelegate::GetOwnerBridge() const { + return bridge_; +} + } // namespace flutter diff --git a/shell/platform/common/flutter_platform_node_delegate.h b/shell/platform/common/flutter_platform_node_delegate.h index 3a562d1c345a0..f28cd9181e3bc 100644 --- a/shell/platform/common/flutter_platform_node_delegate.h +++ b/shell/platform/common/flutter_platform_node_delegate.h @@ -39,6 +39,19 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase { public: virtual ~OwnerBridge() = default; + //--------------------------------------------------------------------------- + /// @brief Gets the rectangular bounds of the ax node relative to + /// global coordinate + /// + /// @param[in] node The ax node to look up. + /// @param[in] offscreen the bool reference to hold the result whether + /// the ax node is outside of its ancestors' bounds. + /// @param[in] clip_bounds whether to clip the result if the ax node cannot + /// be fully contained in its ancestors' bounds. + virtual gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node, + bool& offscreen, + bool clip_bounds) = 0; + protected: friend class FlutterPlatformNodeDelegate; @@ -78,19 +91,6 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase { /// /// @param[in] node_id The id of the focused node. virtual void SetLastFocusedId(AccessibilityNodeId node_id) = 0; - - //--------------------------------------------------------------------------- - /// @brief Gets the rectangular bounds of the ax node relative to - /// global coordinate - /// - /// @param[in] node The ax node to look up. - /// @param[in] offscreen the bool reference to hold the result whether - /// the ax node is outside of its ancestors' bounds. - /// @param[in] clip_bounds whether to clip the result if the ax node cannot - /// be fully contained in its ancestors' bounds. - virtual gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node, - bool& offscreen, - bool clip_bounds) = 0; }; FlutterPlatformNodeDelegate(); @@ -129,11 +129,18 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase { /// Subclasses must call super. virtual void Init(std::weak_ptr bridge, ui::AXNode* node); - protected: //------------------------------------------------------------------------------ - /// @brief Gets the underlying ax node for this accessibility node. + /// @brief Gets the underlying ax node for this platform node delegate. ui::AXNode* GetAXNode() const; + //------------------------------------------------------------------------------ + /// @brief Gets the owner of this platform node delegate. This is useful + /// when you want to get the information about surrounding nodes + /// of this platform node delegate, e.g. the global rect of this + /// platform node delegate. This pointer is only safe in the + /// platform thread. + std::weak_ptr GetOwnerBridge() const; + private: ui::AXNode* ax_node_; std::weak_ptr bridge_; diff --git a/shell/platform/common/flutter_platform_node_delegate_unittests.cc b/shell/platform/common/flutter_platform_node_delegate_unittests.cc index d4753265f4b07..d6aecb2c61cc0 100644 --- a/shell/platform/common/flutter_platform_node_delegate_unittests.cc +++ b/shell/platform/common/flutter_platform_node_delegate_unittests.cc @@ -174,5 +174,51 @@ TEST(FlutterPlatformNodeDelegateTest, canCalculateOffScreenBoundsCorrectly) { EXPECT_EQ(result, ui::AXOffscreenResult::kOffscreen); } +TEST(FlutterPlatformNodeDelegateTest, canUseOwnerBridge) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + root.rect = {0, 0, 100, 100}; // LTRB + root.transform = {1, 0, 0, 0, 1, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + child1.rect = {0, 0, 50, 50}; // LTRB + child1.transform = {0.5, 0, 0, 0, 0.5, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + auto owner_bridge = child1_node->GetOwnerBridge().lock(); + + bool result; + gfx::RectF bounds = owner_bridge->RelativeToGlobalBounds( + child1_node->GetAXNode(), result, true); + EXPECT_EQ(bounds.x(), 0); + EXPECT_EQ(bounds.y(), 0); + EXPECT_EQ(bounds.width(), 25); + EXPECT_EQ(bounds.height(), 25); + EXPECT_EQ(result, false); +} + } // namespace testing } // namespace flutter diff --git a/shell/platform/common/test_accessibility_bridge.cc b/shell/platform/common/test_accessibility_bridge.cc index 7127989a8d4f7..af92c638660f3 100644 --- a/shell/platform/common/test_accessibility_bridge.cc +++ b/shell/platform/common/test_accessibility_bridge.cc @@ -6,7 +6,7 @@ namespace flutter { -std::unique_ptr +std::shared_ptr TestAccessibilityBridgeDelegate::CreateFlutterPlatformNodeDelegate() { return std::make_unique(); }; diff --git a/shell/platform/common/test_accessibility_bridge.h b/shell/platform/common/test_accessibility_bridge.h index 427b88c276cbb..8f618409fc5ba 100644 --- a/shell/platform/common/test_accessibility_bridge.h +++ b/shell/platform/common/test_accessibility_bridge.h @@ -19,8 +19,8 @@ class TestAccessibilityBridgeDelegate void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override; - std::unique_ptr - CreateFlutterPlatformNodeDelegate(); + std::shared_ptr + CreateFlutterPlatformNodeDelegate() override; std::vector accessibilitiy_events; std::vector performed_actions; diff --git a/shell/platform/darwin/macos/BUILD.gn b/shell/platform/darwin/macos/BUILD.gn index b868ff413f314..97a89b5fa5952 100644 --- a/shell/platform/darwin/macos/BUILD.gn +++ b/shell/platform/darwin/macos/BUILD.gn @@ -106,6 +106,8 @@ source_set("flutter_framework_source") { "framework/Source/FlutterSurfaceManager.mm", "framework/Source/FlutterTextInputPlugin.h", "framework/Source/FlutterTextInputPlugin.mm", + "framework/Source/FlutterTextInputSemanticsObject.h", + "framework/Source/FlutterTextInputSemanticsObject.mm", "framework/Source/FlutterTextureRegistrar.h", "framework/Source/FlutterTextureRegistrar.mm", "framework/Source/FlutterView.h", @@ -181,6 +183,7 @@ executable("flutter_desktop_darwin_unittests") { "framework/Source/FlutterOpenGLRendererTest.mm", "framework/Source/FlutterPlatformNodeDelegateMacTest.mm", "framework/Source/FlutterTextInputPluginTest.mm", + "framework/Source/FlutterTextInputSemanticsObjectTest.mm", "framework/Source/FlutterViewControllerTest.mm", "framework/Source/FlutterViewControllerTestUtils.h", "framework/Source/FlutterViewControllerTestUtils.mm", diff --git a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h index ea51f93c2cf31..772234b5b606c 100644 --- a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h +++ b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h @@ -10,6 +10,7 @@ #include "flutter/shell/platform/common/accessibility_bridge.h" @class FlutterEngine; +@class FlutterViewController; namespace flutter { @@ -20,8 +21,10 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility public: //--------------------------------------------------------------------------- /// @brief Creates an AccessibilityBridgeMacDelegate. - /// @param[in] flutterEngine The weak reference to the FlutterEngine. - explicit AccessibilityBridgeMacDelegate(__weak FlutterEngine* flutter_engine); + /// @param[in] flutter_engine The weak reference to the FlutterEngine. + /// @param[in] view_controller The weak reference to the FlutterViewController. + explicit AccessibilityBridgeMacDelegate(__weak FlutterEngine* flutter_engine, + __weak FlutterViewController* view_controller); virtual ~AccessibilityBridgeMacDelegate() = default; // |AccessibilityBridge::AccessibilityBridgeDelegate| @@ -33,7 +36,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility fml::MallocMapping data) override; // |AccessibilityBridge::AccessibilityBridgeDelegate| - std::unique_ptr CreateFlutterPlatformNodeDelegate() override; + std::shared_ptr CreateFlutterPlatformNodeDelegate() override; private: /// A wrapper structure to wraps macOS native accessibility events. @@ -64,7 +67,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility //--------------------------------------------------------------------------- /// @brief Whether the given event is in current pending events. - /// @param[in] event_type The event you would like to look up. + /// @param[in] event_type The event to look up. bool HasPendingEvent(ui::AXEventGenerator::Event event) const; //--------------------------------------------------------------------------- @@ -76,6 +79,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility const ui::AXNode& ax_node) const; __weak FlutterEngine* flutter_engine_; + __weak FlutterViewController* view_controller_; }; } // namespace flutter diff --git a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.mm b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.mm index 35746b04e64ed..36659708c84b6 100644 --- a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.mm +++ b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.mm @@ -6,6 +6,7 @@ #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -14,7 +15,6 @@ inline constexpr int32_t kRootNode = 0; // Native mac notifications fired. These notifications are not publicly documented. -// We have to define them ourselves. static NSString* const AccessibilityLoadCompleteNotification = @"AXLoadComplete"; static NSString* const AccessibilityInvalidStatusChangedNotification = @"AXInvalidStatusChanged"; static NSString* const AccessibilityLiveRegionCreatedNotification = @"AXLiveRegionCreated"; @@ -22,13 +22,15 @@ static NSString* const AccessibilityExpandedChanged = @"AXExpandedChanged"; static NSString* const AccessibilityMenuItemSelectedNotification = @"AXMenuItemSelected"; -AccessibilityBridgeMacDelegate::AccessibilityBridgeMacDelegate(__weak FlutterEngine* flutter_engine) - : flutter_engine_(flutter_engine) {} +AccessibilityBridgeMacDelegate::AccessibilityBridgeMacDelegate( + __weak FlutterEngine* flutter_engine, + __weak FlutterViewController* view_controller) + : flutter_engine_(flutter_engine), view_controller_(view_controller) {} void AccessibilityBridgeMacDelegate::OnAccessibilityEvent( ui::AXEventGenerator::TargetedEvent targeted_event) { if (!flutter_engine_.viewController.viewLoaded || !flutter_engine_.viewController.view.window) { - // We don't need to send accessibility events if the there is no view or window. + // Don't need to send accessibility events if the there is no view or window. return; } ui::AXNode* ax_node = targeted_event.node; @@ -66,8 +68,8 @@ .user_info = nil, }); } else if (ax_node.data().role == ax::mojom::Role::kTextFieldWithComboBox) { - // Even though the selected item in the combo box has changed, we don't - // want to post a focus change because this will take the focus out of + // Even though the selected item in the combo box has changed, don't + // post a focus change because this will take the focus out of // the combo box where the user might be typing. events.push_back({ .name = NSAccessibilitySelectedChildrenChangedNotification, @@ -75,7 +77,7 @@ .user_info = nil, }); } - // In all other cases we should post + // In all other cases, this delegate should post // |NSAccessibilityFocusedUIElementChangedNotification|, but this is // handled elsewhere. break; @@ -121,6 +123,17 @@ } break; case ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED: { + id focused = mac_platform_node_delegate->GetFocus(); + if ([focused isKindOfClass:[FlutterTextField class]]) { + // If it is a text field, the selection notifications are handled by + // the FlutterTextField directly. Only need to make sure it is the + // first responder. + FlutterTextField* native_text_field = (FlutterTextField*)focused; + if (native_text_field == mac_platform_node_delegate->GetFocus()) { + [native_text_field becomeFirstResponder]; + } + break; + } // This event always fires at root events.push_back({ .name = NSAccessibilitySelectedTextChangedNotification, @@ -152,7 +165,19 @@ .user_info = nil, }); break; - case ui::AXEventGenerator::Event::VALUE_CHANGED: + case ui::AXEventGenerator::Event::VALUE_CHANGED: { + if (ax_node.data().role == ax::mojom::Role::kTextField) { + // If it is a text field, the value change notifications are handled by + // the FlutterTextField directly. Only need to make sure it is the + // first responder. + FlutterTextField* native_text_field = + (FlutterTextField*)mac_platform_node_delegate->GetNativeViewAccessible(); + id focused = mac_platform_node_delegate->GetFocus(); + if (!focused || native_text_field == focused) { + [native_text_field becomeFirstResponder]; + } + break; + } events.push_back({ .name = NSAccessibilityValueChangedNotification, .target = native_node, @@ -170,6 +195,7 @@ } } break; + } case ui::AXEventGenerator::Event::LIVE_REGION_CREATED: events.push_back({ .name = AccessibilityLiveRegionCreatedNotification, @@ -183,7 +209,7 @@ .target = native_node, .user_info = nil, }); - // Voiceover requires a live region changed notification to actually + // VoiceOver requires a live region changed notification to actually // announce the live region. auto live_region_events = MacOSEventsFromAXEvent(ui::AXEventGenerator::Event::LIVE_REGION_CHANGED, ax_node); @@ -347,9 +373,9 @@ [flutter_engine_ dispatchSemanticsAction:action toTarget:target withData:std::move(data)]; } -std::unique_ptr +std::shared_ptr AccessibilityBridgeMacDelegate::CreateFlutterPlatformNodeDelegate() { - return std::make_unique(flutter_engine_); + return std::make_shared(flutter_engine_, view_controller_); } // Private method diff --git a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegateTest.mm b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegateTest.mm index 4bd7166bf358e..da0168be7915b 100644 --- a/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegateTest.mm +++ b/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegateTest.mm @@ -13,8 +13,9 @@ class AccessibilityBridgeMacDelegateSpy : public AccessibilityBridgeMacDelegate { public: - AccessibilityBridgeMacDelegateSpy(__weak FlutterEngine* flutter_engine) - : AccessibilityBridgeMacDelegate(flutter_engine) {} + AccessibilityBridgeMacDelegateSpy(__weak FlutterEngine* flutter_engine, + __weak FlutterViewController* view_controller) + : AccessibilityBridgeMacDelegate(flutter_engine, view_controller) {} std::unordered_map actual_notifications; @@ -73,7 +74,7 @@ void DispatchMacOSNotification(gfx::NativeViewAccessible native_node, bridge->CommitUpdates(); auto platform_node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); - AccessibilityBridgeMacDelegateSpy spy(engine); + AccessibilityBridgeMacDelegateSpy spy(engine, viewController); // Creates a targeted event. ui::AXTree tree; @@ -96,6 +97,13 @@ void DispatchMacOSNotification(gfx::NativeViewAccessible native_node, TEST(AccessibilityBridgeMacDelegateTest, doesNotSendAccessibilityCreateNotificationWhenHeadless) { FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; // Setting up bridge so that the AccessibilityBridgeMacDelegateSpy // can query semantics information from. engine.semanticsEnabled = YES; @@ -118,7 +126,7 @@ void DispatchMacOSNotification(gfx::NativeViewAccessible native_node, bridge->CommitUpdates(); auto platform_node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); - AccessibilityBridgeMacDelegateSpy spy(engine); + AccessibilityBridgeMacDelegateSpy spy(engine, viewController); // Creates a targeted event. ui::AXTree tree; @@ -171,7 +179,7 @@ void DispatchMacOSNotification(gfx::NativeViewAccessible native_node, bridge->CommitUpdates(); auto platform_node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); - AccessibilityBridgeMacDelegateSpy spy(engine); + AccessibilityBridgeMacDelegateSpy spy(engine, viewController); // Creates a targeted event. ui::AXTree tree; diff --git a/shell/platform/darwin/macos/framework/Source/FlutterCompositor.h b/shell/platform/darwin/macos/framework/Source/FlutterCompositor.h index aae5f0a439df5..42997ea464f43 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterCompositor.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterCompositor.h @@ -25,8 +25,8 @@ class FlutterCompositor { // Creates a BackingStore and saves updates the backing_store_out // data with the new BackingStore data. // If the backing store is being requested for the first time - // for a given frame, we do not create a new backing store but - // rather return the backing store associated with the + // for a given frame, this compositor does not create a new backing + // store but rather returns the backing store associated with the // FlutterView's FlutterSurfaceManager. // // Any additional state allocated for the backing store and diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm index 9c6914648b446..5c9c48bd04195 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm @@ -198,10 +198,10 @@ static NSUInteger computeModifierFlagOfInterestMask() { // which is used for the "Apple logo" character (Option-Shift-K on a US // keyboard.) // - // We hereby assume that non-printable function keys are defined from + // Assume that non-printable function keys are defined from // 0xF700 upwards, and printable private keys are defined from 0xF8FF - // downwards. We want to keep the printable private keys, therefore we only - // filter out 0xF700-0xF7FF. + // downwards. This function filters out 0xF700-0xF7FF in order to keep + // the printable private keys. return nullptr; } return [characters UTF8String]; @@ -215,8 +215,8 @@ static NSUInteger computeModifierFlagOfInterestMask() { * The embedder functions only accept C-functions as callbacks, as well as an * arbitrary user_data. In order to send an instance method of * |FlutterEmbedderKeyResponder.handleResponse| to the engine's |SendKeyEvent|, - * we wrap the invocation into a C-function |HandleResponse| and invocation - * context |FlutterKeyPendingResponse|. + * the embedder wraps the invocation into a C-function |HandleResponse| and + * invocation context |FlutterKeyPendingResponse|. * * When this object is sent to the engine's |SendKeyEvent| as |user_data|, it * must be attached with |__bridge_retained|. When this object is parsed @@ -544,7 +544,8 @@ - (void)sendCapsLockTapWithTimestamp:(NSTimeInterval)timestamp // MacOS sends a down *or* an up when CapsLock is tapped, alternatively on // even taps and odd taps. A CapsLock down or CapsLock up should always be // converted to a down *and* an up, and the up should always be a synthesized - // event, since we will never know when the button is released. + // event, since the FlutterEmbedderKeyResponder will never know when the + // button is released. FlutterKeyEvent flutterEvent = { .struct_size = sizeof(FlutterKeyEvent), .timestamp = GetFlutterTimestampFrom(timestamp), diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm b/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm index 58a212a1bb42c..ca10495f48b11 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm @@ -96,7 +96,10 @@ - (instancetype)initWithPlugin:(NSString*)pluginKey flutterEngine:(FlutterEngine } - (NSView*)view { - return _flutterEngine.viewController.view; + if (!_flutterEngine.viewController.viewLoaded) { + [_flutterEngine.viewController loadView]; + } + return _flutterEngine.viewController.flutterView; } - (void)addMethodCallDelegate:(nonnull id)delegate @@ -305,11 +308,18 @@ - (void)loadAOTData:(NSString*)assetsDir { } - (void)setViewController:(FlutterViewController*)controller { - _viewController = controller; - [_renderer setFlutterView:controller.flutterView]; + if (_viewController != controller) { + _viewController = controller; + [_renderer setFlutterView:controller.flutterView]; + + if (_semanticsEnabled && _bridge) { + _bridge->UpdateDelegate( + std::make_unique(self, _viewController)); + } - if (!controller && !_allowHeadlessExecution) { - [self shutDownEngine]; + if (!controller && !_allowHeadlessExecution) { + [self shutDownEngine]; + } } } @@ -417,10 +427,10 @@ - (void)updateDisplayConfig { } - (void)updateWindowMetrics { - if (!_engine) { + if (!_engine || !_viewController.viewLoaded) { return; } - NSView* view = _viewController.view; + NSView* view = _viewController.flutterView; CGRect scaledBounds = [view convertRectToBacking:view.bounds]; CGSize scaledSize = scaledBounds.size; double pixelRatio = view.bounds.size.width == 0 ? 1 : scaledSize.width / view.bounds.size.width; @@ -451,14 +461,15 @@ - (void)setSemanticsEnabled:(BOOL)enabled { return; } _semanticsEnabled = enabled; + // Remove the accessibility children from flutter view before reseting the bridge. + if (!_semanticsEnabled && self.viewController.viewLoaded) { + self.viewController.flutterView.accessibilityChildren = nil; + } if (!_semanticsEnabled && _bridge) { _bridge.reset(); } else if (_semanticsEnabled && !_bridge) { _bridge = std::make_shared( - std::make_unique(self)); - } - if (!_semanticsEnabled) { - self.viewController.view.accessibilityChildren = nil; + std::make_unique(self, self.viewController)); } _embedderAPI.UpdateSemanticsEnabled(_engine, _semanticsEnabled); } @@ -667,15 +678,19 @@ - (void)updateSemanticsCustomActions:(const FlutterSemanticsCustomAction*)action // Custom action with id = kFlutterSemanticsNodeIdBatchEnd indicates this is // the end of the update batch. _bridge->CommitUpdates(); + // Accessibility tree can only be used when the view is loaded. + if (!self.viewController.viewLoaded) { + return; + } // Attaches the accessibility root to the flutter view. auto root = _bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); if (root) { - if ([self.viewController.view.accessibilityChildren count] == 0) { + if ([self.viewController.flutterView.accessibilityChildren count] == 0) { NSAccessibilityElement* native_root = root->GetNativeViewAccessible(); - self.viewController.view.accessibilityChildren = @[ native_root ]; + self.viewController.flutterView.accessibilityChildren = @[ native_root ]; } } else { - self.viewController.view.accessibilityChildren = nil; + self.viewController.flutterView.accessibilityChildren = nil; } return; } diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm index da747aed4839a..99c8c6e4ce4e8 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm @@ -76,6 +76,7 @@ initWithAssetsPath:fixtures ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; [engine setViewController:viewController]; // Enable the semantics. bool enabled_called = false; @@ -128,8 +129,8 @@ update_action_callback(&action_batch_end, (void*)CFBridgingRetain(engine)); // Verify the accessibility tree is attached to the flutter view. - EXPECT_EQ([engine.viewController.view.accessibilityChildren count], 1u); - NSAccessibilityElement* native_root = engine.viewController.view.accessibilityChildren[0]; + EXPECT_EQ([engine.viewController.flutterView.accessibilityChildren count], 1u); + NSAccessibilityElement* native_root = engine.viewController.flutterView.accessibilityChildren[0]; std::string root_label = [native_root.accessibilityLabel UTF8String]; EXPECT_TRUE(root_label == "root"); EXPECT_EQ(native_root.accessibilityRole, NSAccessibilityGroupRole); @@ -139,7 +140,6 @@ EXPECT_TRUE(child1_value == "child 1"); EXPECT_EQ(native_child1.accessibilityRole, NSAccessibilityStaticTextRole); EXPECT_EQ([native_child1.accessibilityChildren count], 0u); - // Disable the semantics. bool semanticsEnabled = true; engine.embedderAPI.UpdateSemanticsEnabled = @@ -150,7 +150,7 @@ engine.semanticsEnabled = NO; EXPECT_FALSE(semanticsEnabled); // Verify the accessibility tree is removed from the view. - EXPECT_EQ([engine.viewController.view.accessibilityChildren count], 0u); + EXPECT_EQ([engine.viewController.flutterView.accessibilityChildren count], 0u); [engine setViewController:nil]; [engine shutDownEngine]; @@ -239,4 +239,95 @@ [engine shutDownEngine]; } +TEST(FlutterEngine, ResetsAccessibilityBridgeWhenSetsNewViewController) { + FlutterEngine* engine = CreateTestEngine(); + // Capture the update callbacks before the embedder API initializes. + auto original_init = engine.embedderAPI.Initialize; + std::function update_node_callback; + std::function update_action_callback; + engine.embedderAPI.Initialize = MOCK_ENGINE_PROC( + Initialize, ([&update_action_callback, &update_node_callback, &original_init]( + size_t version, const FlutterRendererConfig* config, + const FlutterProjectArgs* args, void* user_data, auto engine_out) { + update_node_callback = args->update_semantics_node_callback; + update_action_callback = args->update_semantics_custom_action_callback; + return original_init(version, config, args, user_data, engine_out); + })); + EXPECT_TRUE([engine runWithEntrypoint:@"main"]); + // Set up view controller. + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + // Enable the semantics. + bool enabled_called = false; + engine.embedderAPI.UpdateSemanticsEnabled = + MOCK_ENGINE_PROC(UpdateSemanticsEnabled, ([&enabled_called](auto engine, bool enabled) { + enabled_called = enabled; + return kSuccess; + })); + engine.semanticsEnabled = YES; + EXPECT_TRUE(enabled_called); + // Send flutter semantics updates. + FlutterSemanticsNode root; + root.id = 0; + root.flags = static_cast(0); + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + update_node_callback(&root, (void*)CFBridgingRetain(engine)); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.flags = static_cast(0); + child1.actions = static_cast(0); + child1.text_selection_base = -1; + child1.text_selection_extent = -1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + update_node_callback(&child1, (void*)CFBridgingRetain(engine)); + + FlutterSemanticsNode node_batch_end; + node_batch_end.id = kFlutterSemanticsNodeIdBatchEnd; + update_node_callback(&node_batch_end, (void*)CFBridgingRetain(engine)); + + FlutterSemanticsCustomAction action_batch_end; + action_batch_end.id = kFlutterSemanticsNodeIdBatchEnd; + update_action_callback(&action_batch_end, (void*)CFBridgingRetain(engine)); + + auto native_root = engine.accessibilityBridge.lock()->GetFlutterPlatformNodeDelegateFromID(0); + EXPECT_FALSE(native_root.expired()); + + // Set up a new view controller. + FlutterViewController* newViewController = + [[FlutterViewController alloc] initWithProject:project]; + [newViewController loadView]; + [engine setViewController:newViewController]; + + auto new_native_root = engine.accessibilityBridge.lock()->GetFlutterPlatformNodeDelegateFromID(0); + // The tree is recreated and the old tree will be destroyed. + EXPECT_FALSE(new_native_root.expired()); + EXPECT_TRUE(native_root.expired()); + + [engine setViewController:nil]; + [engine shutDownEngine]; +} + } // namespace flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h b/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h index a5e8c37091b72..ad7ee589dff8d 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h @@ -27,8 +27,8 @@ class FlutterGLCompositor : public FlutterCompositor { // Creates a BackingStore and saves updates the backing_store_out // data with the new BackingStore data. // If the backing store is being requested for the first time - // for a given frame, we do not create a new backing store but - // rather return the backing store associated with the + // for a given frame, this compositor does not create a new backing + // store but rather returns the backing store associated with the // FlutterView's FlutterSurfaceManager. // // Any additional state allocated for the backing store and diff --git a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm index cdb9704f96b0b..9faf00103f6e9 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm @@ -46,7 +46,7 @@ - (void)addSecondaryResponder:(nonnull id)responde } - (void)handleEvent:(nonnull NSEvent*)event { - // Be sure to add a handling method in propagateKeyEvent if you allow more + // Be sure to add a handling method in propagateKeyEvent when allowing more // event types here. if (event.type != NSEventTypeKeyDown && event.type != NSEventTypeKeyUp && event.type != NSEventTypeFlagsChanged) { diff --git a/shell/platform/darwin/macos/framework/Source/FlutterMetalCompositor.h b/shell/platform/darwin/macos/framework/Source/FlutterMetalCompositor.h index e3b7ee5628358..0eeb56f5c0f0c 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterMetalCompositor.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterMetalCompositor.h @@ -21,8 +21,8 @@ class FlutterMetalCompositor : public FlutterCompositor { // backing store. // // If the backing store is being requested for the first time - // for a given frame, we do not create a new backing store but - // rather return the backing store associated with the + // for a given frame, this compositor does not create a new backing + // store but rather returns the backing store associated with the // FlutterView's FlutterSurfaceManager. // // Any additional state allocated for the backing store and diff --git a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h index 0ef94fbba259b..3077ab2fef6d2 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h @@ -19,9 +19,13 @@ namespace flutter { /// AXPlatformNodeMac to manage the macOS-specific accessibility objects. class FlutterPlatformNodeDelegateMac : public FlutterPlatformNodeDelegate { public: - explicit FlutterPlatformNodeDelegateMac(__weak FlutterEngine* engine); + explicit FlutterPlatformNodeDelegateMac( + __weak FlutterEngine* engine, + __weak FlutterViewController* view_controller); virtual ~FlutterPlatformNodeDelegateMac(); + void Init(std::weak_ptr bridge, ui::AXNode* node) override; + //--------------------------------------------------------------------------- /// @brief Gets the live region text of this node in UTF-8 format. This /// is useful to determine the changes in between semantics @@ -46,6 +50,7 @@ class FlutterPlatformNodeDelegateMac : public FlutterPlatformNodeDelegate { private: ui::AXPlatformNode* ax_platform_node_; __weak FlutterEngine* engine_; + __weak FlutterViewController* view_controller_; gfx::RectF ConvertBoundsFromLocalToScreen( const gfx::RectF& local_bounds) const; diff --git a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.mm b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.mm index d48f44ece3d52..ab95291d504c9 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.mm @@ -5,8 +5,9 @@ #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h" #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h" -#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" #include "flutter/shell/platform/common/accessibility_bridge.h" #include "flutter/third_party/accessibility/ax/ax_action_data.h" @@ -15,16 +16,27 @@ #include "flutter/third_party/accessibility/ax/platform/ax_platform_node_base.h" #include "flutter/third_party/accessibility/base/string_utils.h" #include "flutter/third_party/accessibility/gfx/geometry/rect_conversions.h" +#include "flutter/third_party/accessibility/gfx/mac/coordinate_conversion.h" namespace flutter { // namespace -FlutterPlatformNodeDelegateMac::FlutterPlatformNodeDelegateMac(FlutterEngine* engine) - : engine_(engine) { - ax_platform_node_ = ui::AXPlatformNode::Create(this); +FlutterPlatformNodeDelegateMac::FlutterPlatformNodeDelegateMac( + __weak FlutterEngine* engine, + __weak FlutterViewController* view_controller) + : engine_(engine), view_controller_(view_controller) {} + +void FlutterPlatformNodeDelegateMac::Init(std::weak_ptr bridge, ui::AXNode* node) { + FlutterPlatformNodeDelegate::Init(bridge, node); + if (GetData().IsTextField()) { + ax_platform_node_ = new FlutterTextPlatformNode(this, view_controller_); + } else { + ax_platform_node_ = ui::AXPlatformNode::Create(this); + } NSCAssert(ax_platform_node_, @"Failed to create platform node."); } FlutterPlatformNodeDelegateMac::~FlutterPlatformNodeDelegateMac() { + // Destroy() also calls delete on itself. ax_platform_node_->Destroy(); } @@ -37,7 +49,8 @@ gfx::NativeViewAccessible parent = FlutterPlatformNodeDelegate::GetParent(); if (!parent) { NSCAssert(engine_, @"Flutter engine should not be deallocated"); - return engine_.viewController.view; + NSCAssert(engine_.viewController.viewLoaded, @"Flutter view must be loaded"); + return engine_.viewController.flutterView; } return parent; } @@ -88,17 +101,18 @@ NSMakeRect(local_bounds.x(), local_bounds.y(), local_bounds.width(), local_bounds.height()); // The macOS XY coordinates start at bottom-left and increase toward top-right, // which is different from the Flutter's XY coordinates that start at top-left - // increasing to bottom-right. Therefore, We need to flip the y coordinate when - // we convert from flutter coordinates to macOS coordinates. + // increasing to bottom-right. Therefore, this method needs to flip the y coordinate when + // it converts the bounds from flutter coordinates to macOS coordinates. ns_local_bounds.origin.y = -ns_local_bounds.origin.y - ns_local_bounds.size.height; - __strong FlutterEngine* strong_engine = engine_; - NSCAssert(strong_engine, @"Flutter engine should not be deallocated"); + + NSCAssert(engine_, @"Flutter engine should not be deallocated"); + NSCAssert(engine_.viewController.viewLoaded, @"Flutter view must be loaded."); NSRect ns_view_bounds = - [strong_engine.viewController.view convertRectFromBacking:ns_local_bounds]; - NSRect ns_window_bounds = [strong_engine.viewController.view convertRect:ns_view_bounds - toView:nil]; + [engine_.viewController.flutterView convertRectFromBacking:ns_local_bounds]; + NSRect ns_window_bounds = [engine_.viewController.flutterView convertRect:ns_view_bounds + toView:nil]; NSRect ns_screen_bounds = - [[strong_engine.viewController.view window] convertRectToScreen:ns_window_bounds]; + [[engine_.viewController.flutterView window] convertRectToScreen:ns_window_bounds]; gfx::RectF screen_bounds(ns_screen_bounds.origin.x, ns_screen_bounds.origin.y, ns_screen_bounds.size.width, ns_screen_bounds.size.height); return screen_bounds; @@ -106,8 +120,8 @@ gfx::RectF FlutterPlatformNodeDelegateMac::ConvertBoundsFromScreenToGlobal( const gfx::RectF& screen_bounds) const { - // The voiceover seems to only accept bounds that are relative to primary screen. - // Thus, we use [[NSScreen screens] firstObject] instead of [NSScreen mainScreen]. + // The VoiceOver seems to only accept bounds that are relative to primary screen. + // Thus, this method uses [[NSScreen screens] firstObject] instead of [NSScreen mainScreen]. NSScreen* screen = [[NSScreen screens] firstObject]; NSRect primary_screen_bounds = [screen frame]; // The screen is flipped against y axis. diff --git a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMacTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMacTest.mm index 067904b4b73cf..c3ac3fcba397e 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMacTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMacTest.mm @@ -3,13 +3,16 @@ // found in the LICENSE file. #include "flutter/testing/testing.h" -#include "flutter/shell/platform/common/accessibility_bridge.h" #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterEngine.h" #import "flutter/shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" + +#include "flutter/shell/platform/common/accessibility_bridge.h" #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h" #include "flutter/third_party/accessibility/ax/ax_action_data.h" @@ -208,4 +211,84 @@ [engine shutDownEngine]; } +TEST(FlutterPlatformNodeDelegateMac, TextFieldUsesFlutterTextField) { + FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + viewController.textInputPlugin.string = @"textfield"; + // Creates a NSWindow so that the native text field can become first responder. + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 800, 600) + styleMask:NSBorderlessWindowMask + backing:NSBackingStoreBuffered + defer:NO]; + window.contentView = viewController.view; + engine.semanticsEnabled = YES; + + auto bridge = engine.accessibilityBridge.lock(); + // Initialize ax node data. + FlutterSemanticsNode root; + root.id = 0; + root.flags = static_cast(0); + root.actions = static_cast(0); + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + root.rect = {0, 0, 100, 100}; // LTRB + root.transform = {1, 0, 0, 0, 1, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + double rectSize = 50; + double transformFactor = 0.5; + + FlutterSemanticsNode child1; + child1.id = 1; + child1.flags = FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField; + child1.actions = static_cast(0); + child1.label = ""; + child1.hint = ""; + child1.value = "textfield"; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.text_selection_base = -1; + child1.text_selection_extent = -1; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + child1.rect = {0, 0, rectSize, rectSize}; // LTRB + child1.transform = {transformFactor, 0, 0, 0, transformFactor, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + + auto child_platform_node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + // Verify the accessibility attribute matches. + id native_accessibility = child_platform_node_delegate->GetNativeViewAccessible(); + EXPECT_EQ([native_accessibility isKindOfClass:[FlutterTextField class]], YES); + FlutterTextField* native_text_field = (FlutterTextField*)native_accessibility; + + NSView* view = viewController.flutterView; + CGRect scaledBounds = [view convertRectToBacking:view.bounds]; + CGSize scaledSize = scaledBounds.size; + double pixelRatio = view.bounds.size.width == 0 ? 1 : scaledSize.width / view.bounds.size.width; + + double expectedFrameSize = rectSize * transformFactor / pixelRatio; + EXPECT_EQ(NSEqualRects(native_text_field.frame, NSMakeRect(0, 600 - expectedFrameSize, + expectedFrameSize, expectedFrameSize)), + YES); + // The text of TextInputPlugin only starts syncing editing state to the + // native text field when it becomes the first responder. + [native_text_field becomeFirstResponder]; + EXPECT_EQ([native_text_field.stringValue isEqualToString:@"textfield"], YES); +} + } // flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.h b/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.h index 4a621c4170fce..88442dcd26cfa 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.h @@ -30,7 +30,7 @@ * Flow during window resize * * 1. Platform thread calls [synchronizer beginResize:notify:] - * This will hold the platform thread until we're ready to display contents. + * This will hold the platform thread until the raster thread is ready to display contents. * 2. Raster thread calls [synchronizer shouldEnsureSurfaceForSize:] with target size * This will return false for any size other than target size * 3. Raster thread calls [synchronizer requestCommit] @@ -45,8 +45,8 @@ * This will invoke [delegate flush:] on raster thread and * [delegate commit:] on platform thread. The requestCommit call will be blocked * until this is done. This is necessary to ensure that rasterizer won't start - * rasterizing next frame before we flipped the surface, which must be performed - * on platform thread + * rasterizing next frame before the FlutterSurfaceManager flipped the surface, + * which must be performed on platform thread. */ @interface FlutterResizeSynchronizer : NSObject diff --git a/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.mm b/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.mm index 82ea1cc8a2b32..6e4873d668e19 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterResizeSynchronizer.mm @@ -17,8 +17,8 @@ @interface FlutterResizeSynchronizer () { // Used to block [requestCommit]. std::condition_variable _condBlockRequestCommit; - // Whether a frame was received; We don't block platform thread during resize - // until we know that framework is running and producing frames + // Whether a frame was received; the synchronizer doesn't block platform thread during resize + // until it knows that framework is running and producing frames BOOL _receivedFirstFrame; // If NO, requestCommit calls are ignored until shouldEnsureSurfaceForSize is called with diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h index 39a98d319aff1..a189d54b28cbf 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h @@ -8,6 +8,8 @@ #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterKeySecondaryResponder.h" +@class FlutterTextField; + /** * A plugin to handle text input. * @@ -16,14 +18,34 @@ * * This is not an FlutterPlugin since it needs access to FlutterViewController internals, so needs * to be managed differently. + * + * When accessibility is on, accessibility bridge creates a NSTextField, i.e. FlutterTextField, + * for every text field in the Flutter. This plugin acts as a field editor for those NSTextField[s]. + */ +@interface FlutterTextInputPlugin : NSTextView + +/** + * The NSTextField that currently has this plugin as its field editor. + * + * Must be nil if accessibility is off. */ -@interface FlutterTextInputPlugin : NSObject +@property(nonatomic, weak) FlutterTextField* client; /** * Initializes a text input plugin that coordinates key event handling with |viewController|. */ - (instancetype)initWithViewController:(FlutterViewController*)viewController; +/** + * Whether this plugin is the first responder of this NSWindow. + * + * When accessibility is on, this plugin is set as the first responder to act as the field + * editor for FlutterTextFields. + * + * Returns false if accessibility is off. + */ +- (BOOL)isFirstResponder; + @end // Private methods made visible for testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm index 71ed1f655da2e..dcf340df50c30 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm @@ -11,6 +11,8 @@ #include "flutter/shell/platform/common/text_input_model.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterCodecs.h" +#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" static NSString* const kTextInputChannel = @"flutter/textinput"; @@ -71,7 +73,7 @@ typedef NS_ENUM(NSUInteger, FlutterTextAffinity) { /** * Private properties of FlutterTextInputPlugin. */ -@interface FlutterTextInputPlugin () +@interface FlutterTextInputPlugin () /** * A text input context, representing a connection to the Cocoa text input system. @@ -133,6 +135,20 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result; */ - (void)setEditingState:(NSDictionary*)state; +/** + * Informs the Flutter framework of changes to the text input model's state. + */ +- (void)updateEditState; + +/** + * Updates the stringValue and selectedRange that stored in the NSTextView interface + * that this plugin inherits from. + * + * If there is a FlutterTextField uses this plugin as its field editor, this method + * will update the stringValue and selectedRange through the API of the FlutterTextField. + */ +- (void)updateTextAndSelection; + @end @implementation FlutterTextInputPlugin { @@ -153,15 +169,22 @@ @implementation FlutterTextInputPlugin { } - (instancetype)initWithViewController:(FlutterViewController*)viewController { - self = [super init]; + // The view needs a non-zero frame. + self = [super initWithFrame:NSMakeRect(0, 0, 1, 1)]; if (self != nil) { + _flutterViewController = viewController; _channel = [FlutterMethodChannel methodChannelWithName:kTextInputChannel binaryMessenger:viewController.engine.binaryMessenger codec:[FlutterJSONMethodCodec sharedInstance]]; _shown = FALSE; - __weak FlutterTextInputPlugin* weakSelf = self; + // NSTextView does not support _weak reference, so this class has to + // use __unsafe_unretained and manage the reference by itself. + // + // Since the dealloc removes the handler, the pointer should + // be valid if the handler is ever called. + __unsafe_unretained FlutterTextInputPlugin* unsafeSelf = self; [_channel setMethodCallHandler:^(FlutterMethodCall* call, FlutterResult result) { - [weakSelf handleMethodCall:call result:result]; + [unsafeSelf handleMethodCall:call result:result]; }]; _textInputContext = [[NSTextInputContext alloc] initWithClient:self]; _previouslyPressedFlags = 0; @@ -176,6 +199,17 @@ - (instancetype)initWithViewController:(FlutterViewController*)viewController { return self; } +- (BOOL)isFirstResponder { + if (!self.flutterViewController.viewLoaded) { + return false; + } + return [self.flutterViewController.view.window firstResponder] == self; +} + +- (void)dealloc { + [_channel setMethodCallHandler:nil]; +} + #pragma mark - Private - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result { @@ -285,11 +319,10 @@ - (void)setEditingState:(NSDictionary*)state { state[kComposingBaseKey], state[kComposingExtentKey], _activeModel->composing_range()); size_t cursor_offset = selected_range.base() - composing_range.start(); _activeModel->SetComposingRange(composing_range, cursor_offset); + [_client becomeFirstResponder]; + [self updateTextAndSelection]; } -/** - * Informs the Flutter framework of changes to the text input model's state. - */ - (void)updateEditState { if (_activeModel == nullptr) { return; @@ -313,6 +346,25 @@ - (void)updateEditState { }; [_channel invokeMethod:kUpdateEditStateResponseMethod arguments:@[ self.clientID, state ]]; + [self updateTextAndSelection]; +} + +- (void)updateTextAndSelection { + NSAssert(_activeModel != nullptr, @"Flutter text model must not be null."); + NSString* text = @(_activeModel->GetText().data()); + int start = _activeModel->selection().base(); + int extend = _activeModel->selection().extent(); + NSRange selection = NSMakeRange(MIN(start, extend), ABS(start - extend)); + // There may be a native text field client if VoiceOver is on. + // In this case, this plugin has to update text and selection through + // the client in order for VoiceOver to announce the text editing + // properly. + if (_client) { + [_client updateString:text withSelection:selection]; + } else { + self.string = text; + [self setSelectedRange:selection]; + } } #pragma mark - @@ -340,6 +392,69 @@ - (BOOL)handleKeyEvent:(NSEvent*)event { return [_textInputContext handleEvent:event]; } +#pragma mark - +#pragma mark NSResponder + +- (void)keyDown:(NSEvent*)event { + [self.flutterViewController keyDown:event]; +} + +- (void)keyUp:(NSEvent*)event { + [self.flutterViewController keyUp:event]; +} + +- (BOOL)performKeyEquivalent:(NSEvent*)event { + return [self.flutterViewController performKeyEquivalent:event]; +} + +- (void)flagsChanged:(NSEvent*)event { + [self.flutterViewController flagsChanged:event]; +} + +- (void)mouseDown:(NSEvent*)event { + [self.flutterViewController mouseDown:event]; +} + +- (void)mouseUp:(NSEvent*)event { + [self.flutterViewController mouseUp:event]; +} + +- (void)mouseDragged:(NSEvent*)event { + [self.flutterViewController mouseDragged:event]; +} + +- (void)rightMouseDown:(NSEvent*)event { + [self.flutterViewController rightMouseDown:event]; +} + +- (void)rightMouseUp:(NSEvent*)event { + [self.flutterViewController rightMouseUp:event]; +} + +- (void)rightMouseDragged:(NSEvent*)event { + [self.flutterViewController rightMouseDragged:event]; +} + +- (void)otherMouseDown:(NSEvent*)event { + [self.flutterViewController otherMouseDown:event]; +} + +- (void)otherMouseUp:(NSEvent*)event { + [self.flutterViewController otherMouseUp:event]; +} + +- (void)otherMouseDragged:(NSEvent*)event { + [self.flutterViewController otherMouseDragged:event]; +} + +- (void)mouseMoved:(NSEvent*)event { + [self.flutterViewController mouseMoved:event]; +} + +- (void)scrollWheel:(NSEvent*)event { + [self.flutterViewController scrollWheel:event]; +} + #pragma mark - #pragma mark NSTextInputClient @@ -351,8 +466,7 @@ - (void)insertText:(id)string replacementRange:(NSRange)range { if (range.location != NSNotFound) { // The selected range can actually have negative numbers, since it can start // at the end of the range if the user selected the text going backwards. - // NSRange uses NSUIntegers, however, so we have to cast them to know if the - // selection is reversed or not. + // Cast to a signed type to determine whether or not the selection is reversed. long signedLength = static_cast(range.length); long location = range.location; long textLength = _activeModel->text_range().end(); @@ -422,14 +536,6 @@ - (void)unmarkText { [self updateEditState]; } -- (NSRange)selectedRange { - if (_activeModel == nullptr) { - return NSMakeRange(NSNotFound, 0); - } - return NSMakeRange(_activeModel->selection().base(), - _activeModel->selection().extent() - _activeModel->selection().base()); -} - - (NSRange)markedRange { if (_activeModel == nullptr) { return NSMakeRange(NSNotFound, 0); @@ -461,6 +567,9 @@ - (NSAttributedString*)attributedSubstringForProposedRange:(NSRange)range } - (NSRect)firstRectForCharacterRange:(NSRange)range actualRange:(NSRangePointer)actualRange { + if (!self.flutterViewController.viewLoaded) { + return CGRectZero; + } // This only determines position of caret instead of any arbitrary range, but it's enough // to properly position accent selection popup if (CATransform3DIsAffine(_editableTransform) && !CGRectEqualToRect(_caretRect, CGRectNull)) { @@ -468,10 +577,10 @@ - (NSRect)firstRectForCharacterRange:(NSRange)range actualRange:(NSRangePointer) CGRectApplyAffineTransform(_caretRect, CATransform3DGetAffineTransform(_editableTransform)); // convert to window coordinates - rect = [self.flutterViewController.view convertRect:rect toView:nil]; + rect = [self.flutterViewController.flutterView convertRect:rect toView:nil]; // convert to screen coordinates - return [self.flutterViewController.view.window convertRectToScreen:rect]; + return [self.flutterViewController.flutterView.window convertRectToScreen:rect]; } else { return CGRectZero; } diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm index 0eeb2d6bc7d72..b7b8c31fa4c54 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm @@ -4,12 +4,30 @@ #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" #import #import "flutter/testing/testing.h" +@interface FlutterTextFieldMock : FlutterTextField + +@property(nonatomic) NSString* lastUpdatedString; +@property(nonatomic) NSRange lastUpdatedSelection; + +@end + +@implementation FlutterTextFieldMock + +- (void)updateString:(NSString*)string withSelection:(NSRange)selection { + _lastUpdatedString = string; + _lastUpdatedSelection = selection; +} + +@end + @interface FlutterInputPluginTestObjc : NSObject - (bool)testEmptyCompositionRange; @end @@ -87,7 +105,7 @@ - (bool)testFirstRectForCharacterRange { OCMStub( // NOLINT(google-objc-avoid-throwing-exception) [engineMock binaryMessenger]) .andReturn(binaryMessengerMock); - id controllerMock = OCMClassMock([FlutterViewController class]); + FlutterViewController* controllerMock = OCMClassMock([FlutterViewController class]); OCMStub( // NOLINT(google-objc-avoid-throwing-exception) [controllerMock engine]) .andReturn(engineMock); @@ -97,7 +115,10 @@ - (bool)testFirstRectForCharacterRange { [viewMock bounds]) .andReturn(NSMakeRect(0, 0, 200, 200)); OCMStub( // NOLINT(google-objc-avoid-throwing-exception) - [controllerMock view]) + controllerMock.viewLoaded) + .andReturn(YES); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [controllerMock flutterView]) .andReturn(viewMock); id windowMock = OCMClassMock([NSWindow class]); @@ -159,6 +180,17 @@ - (bool)testFirstRectForCharacterRange { namespace flutter::testing { +namespace { +// Allocates and returns an engine configured for the text fixture resource configuration. +FlutterEngine* CreateTestEngine() { + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + return [[FlutterEngine alloc] initWithName:@"test" project:project allowHeadlessExecution:true]; +} +} // namespace + TEST(FlutterTextInputPluginTest, TestEmptyCompositionRange) { ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] testEmptyCompositionRange]); } @@ -167,4 +199,100 @@ - (bool)testFirstRectForCharacterRange { ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] testFirstRectForCharacterRange]); } +TEST(FlutterTextInputPluginTest, CanWorkWithFlutterTextField) { + FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + // Create a NSWindow so that the native text field can become first responder. + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 800, 600) + styleMask:NSBorderlessWindowMask + backing:NSBackingStoreBuffered + defer:NO]; + window.contentView = viewController.view; + + engine.semanticsEnabled = YES; + + auto bridge = engine.accessibilityBridge.lock(); + FlutterPlatformNodeDelegateMac delegate(engine, viewController); + ui::AXTree tree; + ui::AXNode ax_node(&tree, nullptr, 0, 0); + ui::AXNodeData node_data; + node_data.SetValue("initial text"); + ax_node.SetData(node_data); + delegate.Init(engine.accessibilityBridge, &ax_node); + FlutterTextPlatformNode text_platform_node(&delegate, viewController); + + FlutterTextFieldMock* mockTextField = + [[FlutterTextFieldMock alloc] initWithPlatformNode:&text_platform_node + fieldEditor:viewController.textInputPlugin]; + [viewController.view addSubview:mockTextField]; + [mockTextField becomeFirstResponder]; + + NSDictionary* arguments = @{ + @"inputAction" : @"action", + @"inputType" : @{@"name" : @"inputName"}, + }; + FlutterMethodCall* methodCall = [FlutterMethodCall methodCallWithMethodName:@"TextInput.setClient" + arguments:@[ @(1), arguments ]]; + FlutterResult result = ^(id result) { + }; + [viewController.textInputPlugin handleMethodCall:methodCall result:result]; + + arguments = @{ + @"text" : @"new text", + @"selectionBase" : @(1), + @"selectionExtent" : @(2), + @"composingBase" : @(-1), + @"composingExtent" : @(-1), + }; + + methodCall = [FlutterMethodCall methodCallWithMethodName:@"TextInput.setEditingState" + arguments:arguments]; + [viewController.textInputPlugin handleMethodCall:methodCall result:result]; + EXPECT_EQ([mockTextField.lastUpdatedString isEqualToString:@"new text"], YES); + EXPECT_EQ(NSEqualRanges(mockTextField.lastUpdatedSelection, NSMakeRange(1, 1)), YES); +} + +TEST(FlutterTextInputPluginTest, CanNotBecomeResponderIfNoViewController) { + FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + // Creates a NSWindow so that the native text field can become first responder. + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 800, 600) + styleMask:NSBorderlessWindowMask + backing:NSBackingStoreBuffered + defer:NO]; + window.contentView = viewController.view; + + engine.semanticsEnabled = YES; + + auto bridge = engine.accessibilityBridge.lock(); + FlutterPlatformNodeDelegateMac delegate(engine, viewController); + ui::AXTree tree; + ui::AXNode ax_node(&tree, nullptr, 0, 0); + ui::AXNodeData node_data; + node_data.SetValue("initial text"); + ax_node.SetData(node_data); + delegate.Init(engine.accessibilityBridge, &ax_node); + FlutterTextPlatformNode text_platform_node(&delegate, viewController); + + FlutterTextField* textField = text_platform_node.GetNativeViewAccessible(); + EXPECT_EQ([textField becomeFirstResponder], YES); + // Removes view controller. + [engine setViewController:nil]; + FlutterTextPlatformNode text_platform_node_no_controller(&delegate, nil); + textField = text_platform_node_no_controller.GetNativeViewAccessible(); + EXPECT_EQ([textField becomeFirstResponder], NO); +} + } // namespace flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h new file mode 100644 index 0000000000000..759581889d8d4 --- /dev/null +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h @@ -0,0 +1,92 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import + +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformNodeDelegateMac.h" + +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node_base.h" + +@class FlutterTextField; +@class FlutterTextInputPlugin; + +namespace flutter { + +//------------------------------------------------------------------------------ +/// The ax platform node for a text field. +class FlutterTextPlatformNode : public ui::AXPlatformNodeBase { + public: + //--------------------------------------------------------------------------- + /// @brief Creates a FlutterTextPlatformNode that uses a + /// FlutterTextField as its NativeViewAccessible. + /// @param[in] delegate The delegate that provides accessibility + /// data. + /// @param[in] view_controller The view_controller that is used for querying + /// the information about FlutterView and + /// FlutterTextInputPlugin. + explicit FlutterTextPlatformNode(FlutterPlatformNodeDelegate* delegate, + __weak FlutterViewController* view_controller); + ~FlutterTextPlatformNode() override; + + //------------------------------------------------------------------------------ + /// @brief Gets the frame of this platform node relative to the view of + /// FlutterViewController. This is used by the FlutterTextField to get its + /// frame rect because the FlutterTextField is a subview of the + /// FlutterViewController.view. + NSRect GetFrame(); + + // |ui::AXPlatformNodeBase| + gfx::NativeViewAccessible GetNativeViewAccessible() override; + + private: + FlutterTextField* appkit_text_field_; + __weak FlutterViewController* view_controller_; + + //------------------------------------------------------------------------------ + /// @brief Ensures the FlutterTextField is attached to the FlutterView. This + /// method returns true if the text field is succesfully attached. If + /// this method returns false, that means the FlutterTextField could not + /// be attached to the FlutterView. This can happen when the FlutterEngine + /// does not have a FlutterViewController or the FlutterView is not loaded + /// yet. + bool EnsureAttachedToView(); + + //------------------------------------------------------------------------------ + /// @brief Detaches the FlutterTextField from the FlutterView if it is not + /// already detached. + void EnsureDetachedFromView(); +}; + +} // namespace flutter + +/** + * An NSTextField implementation that represents the NativeViewAccessible for the + * FlutterTextPlatformNode + * + * The NSAccessibility protocol does not provide full support for text editing. This + * appkit text field is used to get around this problem. The FlutterTextPlatformNode + * creates a hidden FlutterTextField, since VoiceOver only provides text editing + * announcements for NSTextField subclasses. + * + * All of the text editing events in this native text field are redirected to the + * FlutterTextInputPlugin. + */ +@interface FlutterTextField : NSTextField + +/** + * Initializes a FlutterTextField that uses the FlutterTextInputPlugin as its field editor. + * The text field redirects all of the text editing events to the FlutterTextInputPlugin. + */ +- (instancetype)initWithPlatformNode:(flutter::FlutterTextPlatformNode*)node + fieldEditor:(FlutterTextInputPlugin*)plugin; + +/** + * Updates the string value and the selection of this text field. + * + * Calling this method is necessary for macOS to get notified about string and selection + * changes. + */ +- (void)updateString:(NSString*)string withSelection:(NSRange)selection; + +@end diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.mm b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.mm new file mode 100644 index 0000000000000..99294970d0db8 --- /dev/null +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.mm @@ -0,0 +1,217 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" + +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" + +#include "flutter/third_party/accessibility/ax/ax_action_data.h" +#include "flutter/third_party/accessibility/gfx/geometry/rect_conversions.h" +#include "flutter/third_party/accessibility/gfx/mac/coordinate_conversion.h" + +#pragma mark - FlutterTextFieldCell +/** + * A convenient class that can be used to set a custom field editor for an + * NSTextField. + * + * The FlutterTextField uses this class set the FlutterTextInputPlugin as + * its field editor. + */ +@interface FlutterTextFieldCell : NSTextFieldCell + +/** + * Initializes the NSCell for the input NSTextField. + */ +- (instancetype)initWithTextField:(NSTextField*)textField fieldEditor:(NSTextView*)editor; + +@end + +@implementation FlutterTextFieldCell { + NSTextView* _editor; +} + +#pragma mark - Private + +- (instancetype)initWithTextField:(NSTextField*)textField fieldEditor:(NSTextView*)editor { + self = [super initTextCell:textField.stringValue]; + if (self) { + _editor = editor; + [self setControlView:textField]; + // Read-only text fields are sent to the mac embedding as static + // text. This text field must be editable and selectable at this + // point. + self.editable = YES; + self.selectable = YES; + } + return self; +} + +#pragma mark - NSCell + +- (NSTextView*)fieldEditorForView:(NSView*)controlView { + return _editor; +} + +@end + +#pragma mark - FlutterTextField + +@implementation FlutterTextField { + flutter::FlutterTextPlatformNode* _node; + FlutterTextInputPlugin* _plugin; +} + +#pragma mark - Public + +- (instancetype)initWithPlatformNode:(flutter::FlutterTextPlatformNode*)node + fieldEditor:(FlutterTextInputPlugin*)plugin { + self = [super initWithFrame:NSZeroRect]; + if (self) { + _node = node; + _plugin = plugin; + [self setCell:[[FlutterTextFieldCell alloc] initWithTextField:self fieldEditor:plugin]]; + } + return self; +} + +- (void)updateString:(NSString*)string withSelection:(NSRange)selection { + NSAssert(_plugin.client == self, + @"Can't update FlutterTextField when it is not the first responder"); + if (![[self stringValue] isEqualToString:string]) { + [self setStringValue:string]; + } + if (!NSEqualRanges(_plugin.selectedRange, selection)) { + [_plugin setSelectedRange:selection]; + } +} + +#pragma mark - NSView + +- (NSRect)frame { + return _node->GetFrame(); +} + +#pragma mark - NSAccessibilityProtocol + +- (void)setAccessibilityFocused:(BOOL)isFocused { + [super setAccessibilityFocused:isFocused]; + ui::AXActionData data; + data.action = isFocused ? ax::mojom::Action::kFocus : ax::mojom::Action::kBlur; + _node->GetDelegate()->AccessibilityPerformAction(data); +} + +#pragma mark - NSResponder + +- (BOOL)becomeFirstResponder { + if (!_plugin) { + return NO; + } + if (_plugin.client == self && [_plugin isFirstResponder]) { + // This text field is already the first responder. + return YES; + } + BOOL result = [super becomeFirstResponder]; + if (result) { + _plugin.client = self; + // The default implementation of the becomeFirstResponder will change the + // text editing state. Need to manually set it back. + NSString* textValue = @(_node->GetStringAttribute(ax::mojom::StringAttribute::kValue).data()); + int start = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelStart); + int end = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelEnd); + NSAssert((start >= 0 && end >= 0) || (start == -1 && end == -1), @"selection is invalid"); + NSRange selection; + if (start >= 0 && end >= 0) { + selection = NSMakeRange(MIN(start, end), ABS(end - start)); + } else { + // The native behavior is to place the cursor at the end of the string if + // there is no selection. + selection = NSMakeRange([self stringValue].length, 0); + } + [self updateString:textValue withSelection:selection]; + } + return result; +} + +- (BOOL)resignFirstResponder { + BOOL result = [super resignFirstResponder]; + if (result && _plugin.client == self) { + _plugin.client = nil; + } + return result; +} + +#pragma mark - NSObject + +- (void)dealloc { + [self resignFirstResponder]; +} + +@end + +namespace flutter { + +FlutterTextPlatformNode::FlutterTextPlatformNode(FlutterPlatformNodeDelegate* delegate, + __weak FlutterViewController* view_controller) { + Init(delegate); + view_controller_ = view_controller; + appkit_text_field_ = + [[FlutterTextField alloc] initWithPlatformNode:this + fieldEditor:view_controller.textInputPlugin]; + appkit_text_field_.bezeled = NO; + appkit_text_field_.drawsBackground = NO; + appkit_text_field_.bordered = NO; + appkit_text_field_.focusRingType = NSFocusRingTypeNone; +} + +FlutterTextPlatformNode::~FlutterTextPlatformNode() { + EnsureDetachedFromView(); +} + +gfx::NativeViewAccessible FlutterTextPlatformNode::GetNativeViewAccessible() { + if (EnsureAttachedToView()) { + return appkit_text_field_; + } + return nil; +} + +NSRect FlutterTextPlatformNode::GetFrame() { + if (!view_controller_.viewLoaded) { + return NSZeroRect; + } + FlutterPlatformNodeDelegate* delegate = static_cast(GetDelegate()); + bool offscreen; + auto bridge_ptr = delegate->GetOwnerBridge().lock(); + gfx::RectF bounds = bridge_ptr->RelativeToGlobalBounds(delegate->GetAXNode(), offscreen, true); + + // Converts to NSRect to use NSView rect conversion. + NSRect ns_local_bounds = NSMakeRect(bounds.x(), bounds.y(), bounds.width(), bounds.height()); + // The macOS XY coordinates start at bottom-left and increase toward top-right, + // which is different from the Flutter's XY coordinates that start at top-left + // increasing to bottom-right. Flip the y coordinate to convert from Flutter + // coordinates to macOS coordinates. + ns_local_bounds.origin.y = -ns_local_bounds.origin.y - ns_local_bounds.size.height; + NSRect ns_view_bounds = [view_controller_.flutterView convertRectFromBacking:ns_local_bounds]; + return [view_controller_.flutterView convertRect:ns_view_bounds toView:nil]; +} + +bool FlutterTextPlatformNode::EnsureAttachedToView() { + if (!view_controller_.viewLoaded) { + return false; + } + if ([appkit_text_field_ isDescendantOf:view_controller_.view]) { + return true; + } + [view_controller_.view addSubview:appkit_text_field_ + positioned:NSWindowBelow + relativeTo:view_controller_.flutterView]; + return true; +} + +void FlutterTextPlatformNode::EnsureDetachedFromView() { + [appkit_text_field_ removeFromSuperview]; +} + +} diff --git a/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObjectTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObjectTest.mm new file mode 100644 index 0000000000000..2080cdbdb8fce --- /dev/null +++ b/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObjectTest.mm @@ -0,0 +1,65 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" + +#import +#import "flutter/testing/testing.h" + +namespace flutter::testing { + +namespace { +// Returns an engine configured for the text fixture resource configuration. +FlutterEngine* CreateTestEngine() { + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + return [[FlutterEngine alloc] initWithName:@"test" project:project allowHeadlessExecution:true]; +} +} // namespace + +TEST(FlutterTextInputSemanticsObjectTest, DoesInitialize) { + FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + // Create a NSWindow so that the native text field can become first responder. + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 800, 600) + styleMask:NSBorderlessWindowMask + backing:NSBackingStoreBuffered + defer:NO]; + window.contentView = viewController.view; + + engine.semanticsEnabled = YES; + + auto bridge = engine.accessibilityBridge.lock(); + FlutterPlatformNodeDelegateMac delegate(engine, viewController); + ui::AXTree tree; + ui::AXNode ax_node(&tree, nullptr, 0, 0); + ui::AXNodeData node_data; + node_data.SetValue("initial text"); + ax_node.SetData(node_data); + delegate.Init(engine.accessibilityBridge, &ax_node); + // Verify that a FlutterTextField is attached to the view. + FlutterTextPlatformNode text_platform_node(&delegate, viewController); + id native_accessibility = text_platform_node.GetNativeViewAccessible(); + EXPECT_TRUE([native_accessibility isKindOfClass:[FlutterTextField class]]); + auto subviews = [viewController.view subviews]; + EXPECT_EQ([subviews count], 2u); + EXPECT_TRUE([subviews[0] isKindOfClass:[FlutterTextField class]]); + FlutterTextField* nativeTextField = subviews[0]; + EXPECT_EQ(text_platform_node.GetNativeViewAccessible(), nativeTextField); +} + +} // namespace flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm index 34502503293d1..942ab17461ea0 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm @@ -7,6 +7,7 @@ #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterChannels.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterCodecs.h" +#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h" #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterEngine.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h" @@ -17,7 +18,7 @@ #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMouseCursorPlugin.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterOpenGLRenderer.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterRenderingBackend.h" -#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h" #import "flutter/shell/platform/embedder/embedder.h" @@ -77,6 +78,23 @@ void Reset() { #pragma mark - Private interface declaration. +/** + * FlutterViewWrapper is a convenience class that wraps a FlutterView and provides + * a mechanism to attach AppKit views such as FlutterTextField without affecting + * the accessibility subtree of the wrapped FlutterView itself. + * + * The FlutterViewController uses this class to create its content view. When + * any of the accessibility services (e.g. VoiceOver) is turned on, the accessibility + * bridge creates FlutterTextFields that interact with the service. The bridge has to + * attach the FlutterTextField somewhere in the view hierarchy in order for the + * FlutterTextField to interact correctly with VoiceOver. Those FlutterTextFields + * will be attached to this view so that they won't affect the accessibility subtree + * of FlutterView. + */ +@interface FlutterViewWrapper : NSView + +@end + /** * Private interface declaration for FlutterViewController. */ @@ -175,6 +193,28 @@ - (BOOL)clipboardHasStrings; @end +#pragma mark - FlutterViewWrapper implementation. + +@implementation FlutterViewWrapper { + FlutterView* _flutterView; +} + +- (instancetype)initWithFlutterView:(FlutterView*)view { + self = [super initWithFrame:NSZeroRect]; + if (self) { + _flutterView = view; + view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + [self addSubview:view]; + } + return self; +} + +- (NSArray*)accessibilityChildren { + return @[ _flutterView ]; +} + +@end + #pragma mark - FlutterViewController implementation. @implementation FlutterViewController { @@ -200,9 +240,9 @@ static void CommonInit(FlutterViewController* controller) { allowHeadlessExecution:NO]; } controller->_mouseTrackingMode = FlutterMouseTrackingModeInKeyWindow; - + controller->_textInputPlugin = [[FlutterTextInputPlugin alloc] initWithViewController:controller]; NSNotificationCenter* center = [NSNotificationCenter defaultCenter]; - // macOS fires this private message when voiceover turns on or off. + // macOS fires this private message when VoiceOver turns on or off. [center addObserver:controller selector:@selector(onAccessibilityStatusChanged:) name:EnhancedUserInterfaceNotification @@ -278,7 +318,9 @@ - (void)loadView { } flutterView = [[FlutterView alloc] initWithMainContext:mainContext reshapeListener:self]; } - self.view = flutterView; + FlutterViewWrapper* wrapperView = [[FlutterViewWrapper alloc] initWithFlutterView:flutterView]; + self.view = wrapperView; + _flutterView = flutterView; } - (void)viewDidLoad { @@ -314,12 +356,6 @@ - (void)setMouseTrackingMode:(FlutterMouseTrackingMode)mode { [self configureTrackingArea]; } -#pragma mark - Framework-internal methods - -- (FlutterView*)flutterView { - return static_cast(self.view); -} - #pragma mark - Private methods - (BOOL)launchEngine { @@ -349,8 +385,9 @@ - (void)listenForMetaModifiedKeyUpEvents { handler:^NSEvent*(NSEvent* event) { // Intercept keyUp only for events triggered on the current // view. - if (weakSelf.view && - ([[event window] firstResponder] == weakSelf.view) && + if (weakSelf.viewLoaded && weakSelf.flutterView && + ([[event window] firstResponder] == + weakSelf.flutterView) && ([event modifierFlags] & NSEventModifierFlagCommand) && ([event type] == NSEventTypeKeyUp)) [weakSelf keyUp:event]; @@ -359,7 +396,8 @@ - (void)listenForMetaModifiedKeyUpEvents { } - (void)configureTrackingArea { - if (_mouseTrackingMode != FlutterMouseTrackingModeNone && self.view) { + NSAssert(self.viewLoaded, @"View must be loaded before setting tracking area"); + if (_mouseTrackingMode != FlutterMouseTrackingModeNone && self.flutterView) { NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited | NSTrackingMouseMoved | NSTrackingInVisibleRect | NSTrackingEnabledDuringMouseDrag; switch (_mouseTrackingMode) { @@ -380,9 +418,9 @@ - (void)configureTrackingArea { options:options owner:self userInfo:nil]; - [self.view addTrackingArea:_trackingArea]; + [self.flutterView addTrackingArea:_trackingArea]; } else if (_trackingArea) { - [self.view removeTrackingArea:_trackingArea]; + [self.flutterView removeTrackingArea:_trackingArea]; _trackingArea = nil; } } @@ -406,8 +444,7 @@ - (void)addInternalPlugins { binaryMessenger:_engine.binaryMessenger codec:[FlutterJSONMessageCodec sharedInstance]]]]; - [_keyboardManager - addSecondaryResponder:[[FlutterTextInputPlugin alloc] initWithViewController:self]]; + [_keyboardManager addSecondaryResponder:_textInputPlugin]; _settingsChannel = [FlutterBasicMessageChannel messageChannelWithName:@"flutter/settings" binaryMessenger:_engine.binaryMessenger @@ -429,6 +466,7 @@ - (void)dispatchMouseEvent:(nonnull NSEvent*)event { } - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase { + NSAssert(self.viewLoaded, @"View must be loaded before it handles the mouse event"); // There are edge cases where the system will deliver enter out of order relative to other // events (e.g., drag out and back in, release, then click; mouseDown: will be called before // mouseEntered:). Discard those events, since the add will already have been synthesized. @@ -452,8 +490,8 @@ - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase { [self dispatchMouseEvent:addEvent phase:kAdd]; } - NSPoint locationInView = [self.view convertPoint:event.locationInWindow fromView:nil]; - NSPoint locationInBackingCoordinates = [self.view convertPointToBacking:locationInView]; + NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil]; + NSPoint locationInBackingCoordinates = [self.flutterView convertPointToBacking:locationInView]; FlutterPointerEvent flutterEvent = { .struct_size = sizeof(flutterEvent), .phase = phase, @@ -476,7 +514,7 @@ - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase { CFRelease(source); } } - double scaleFactor = self.view.layer.contentsScale; + double scaleFactor = self.flutterView.layer.contentsScale; flutterEvent.scroll_delta_x = -event.scrollingDeltaX * pixelsPerLine * scaleFactor; flutterEvent.scroll_delta_y = -event.scrollingDeltaY * pixelsPerLine * scaleFactor; } @@ -502,7 +540,20 @@ - (void)onAccessibilityStatusChanged:(NSNotification*)notification { if (!_engine) { return; } - _engine.semanticsEnabled = !!notification.userInfo[EnhancedUserInterfaceKey]; + BOOL enabled = [notification.userInfo[EnhancedUserInterfaceKey] boolValue]; + if (!enabled && self.viewLoaded && [_textInputPlugin isFirstResponder]) { + // The client (i.e. the FlutterTextField) of the textInputPlugin is a sibling + // of the FlutterView. macOS will pick the ancestor to be the next responder + // when the client is removed from the view hierarchy, which is the result of + // turning off semantics. This will cause the keyboard focus to stick at the + // NSWindow. + // + // Since the view controller creates the illustion that the FlutterTextField is + // below the FlutterView in accessibility (See FlutterViewWrapper), it has to + // manually pick the next responder. + [self.view.window makeFirstResponder:_flutterView]; + } + _engine.semanticsEnabled = [notification.userInfo[EnhancedUserInterfaceKey] boolValue]; } - (void)onSettingsChanged:(NSNotification*)notification { @@ -607,6 +658,27 @@ - (void)keyUp:(NSEvent*)event { [_keyboardManager handleEvent:event]; } +- (BOOL)performKeyEquivalent:(NSEvent*)event { + [_keyboardManager handleEvent:event]; + if (event.type == NSEventTypeKeyDown) { + // macOS only sends keydown for performKeyEquivalent, but the Flutter framework + // always expects a keyup for every keydown. Synthesizes a key up event so that + // the Flutter framework continues to work. + NSEvent* synthesizedUp = [NSEvent keyEventWithType:NSEventTypeKeyUp + location:event.locationInWindow + modifierFlags:event.modifierFlags + timestamp:event.timestamp + windowNumber:event.windowNumber + context:event.context + characters:event.characters + charactersIgnoringModifiers:event.charactersIgnoringModifiers + isARepeat:event.isARepeat + keyCode:event.keyCode]; + [_keyboardManager handleEvent:synthesizedUp]; + } + return YES; +} + - (void)flagsChanged:(NSEvent*)event { [_keyboardManager handleEvent:event]; } diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm index fbf01cfbe7dd5..c0cd3ce4f8100 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm @@ -19,6 +19,7 @@ - (bool)testKeyEventsAreSentToFramework; - (bool)testKeyEventsArePropagatedIfNotHandled; - (bool)testKeyEventsAreNotPropagatedIfHandled; - (bool)testFlagsChangedEventsArePropagatedIfNotHandled; +- (bool)testPerformKeyEquivalentSynthesizesKeyUp; + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event callback:(nullable FlutterKeyEventCallback)callback @@ -29,6 +30,15 @@ + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event namespace { +// Allocates and returns an engine configured for the test fixture resource configuration. +FlutterEngine* CreateTestEngine() { + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + return [[FlutterEngine alloc] initWithName:@"test" project:project allowHeadlessExecution:true]; +} + NSResponder* mockResponder() { NSResponder* mock = OCMStrictClassMock([NSResponder class]); OCMStub([mock keyDown:[OCMArg any]]).andDo(nil); @@ -77,6 +87,59 @@ + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event EXPECT_TRUE(value); } +TEST(FlutterViewController, HasViewThatHidesOtherViewsInAccessibility) { + FlutterViewController* viewControllerMock = CreateMockViewController(nil); + + [viewControllerMock loadView]; + auto subViews = [viewControllerMock.view subviews]; + + EXPECT_EQ([subViews count], 1u); + EXPECT_EQ(subViews[0], viewControllerMock.flutterView); + + NSTextField* textField = [[NSTextField alloc] initWithFrame:NSMakeRect(0, 0, 1, 1)]; + [viewControllerMock.view addSubview:textField]; + + subViews = [viewControllerMock.view subviews]; + EXPECT_EQ([subViews count], 2u); + + auto accessibilityChildren = viewControllerMock.view.accessibilityChildren; + // The accessibilityChildren should only contains the FlutterView. + EXPECT_EQ([accessibilityChildren count], 1u); + EXPECT_EQ(accessibilityChildren[0], viewControllerMock.flutterView); +} + +TEST(FlutterViewController, SetsFlutterViewFirstResponderWhenAccessibilityDisabled) { + FlutterEngine* engine = CreateTestEngine(); + NSString* fixtures = @(testing::GetFixturesPath()); + FlutterDartProject* project = [[FlutterDartProject alloc] + initWithAssetsPath:fixtures + ICUDataPath:[fixtures stringByAppendingString:@"/icudtl.dat"]]; + FlutterViewController* viewController = [[FlutterViewController alloc] initWithProject:project]; + [viewController loadView]; + [engine setViewController:viewController]; + // Creates a NSWindow so that sub view can be first responder. + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 800, 600) + styleMask:NSBorderlessWindowMask + backing:NSBackingStoreBuffered + defer:NO]; + window.contentView = viewController.view; + // Attaches FlutterTextInputPlugin to the view; + [viewController.view addSubview:viewController.textInputPlugin]; + // Makes sure the textInputPlugin can be the first responder. + EXPECT_TRUE([window makeFirstResponder:viewController.textInputPlugin]); + EXPECT_EQ([window firstResponder], viewController.textInputPlugin); + // Sends a notification to turn off the accessibility. + NSDictionary* userInfo = @{ + @"AXEnhancedUserInterface" : @(NO), + }; + NSNotification* accessibilityOff = [NSNotification notificationWithName:@"" + object:nil + userInfo:userInfo]; + [viewController onAccessibilityStatusChanged:accessibilityOff]; + // FlutterView becomes the first responder. + EXPECT_EQ([window firstResponder], viewController.flutterView); +} + TEST(FlutterViewControllerTest, TestKeyEventsAreSentToFramework) { ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testKeyEventsAreSentToFramework]); } @@ -94,6 +157,10 @@ + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event [[FlutterViewControllerTestObjC alloc] testFlagsChangedEventsArePropagatedIfNotHandled]); } +TEST(FlutterViewControllerTest, TestPerformKeyEquivalentSynthesizesKeyUp) { + ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testPerformKeyEquivalentSynthesizesKeyUp]); +} + } // namespace flutter::testing @implementation FlutterViewControllerTestObjC @@ -299,6 +366,86 @@ - (bool)testKeyEventsAreNotPropagatedIfHandled { return true; } +- (bool)testPerformKeyEquivalentSynthesizesKeyUp { + id engineMock = OCMClassMock([FlutterEngine class]); + id binaryMessengerMock = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [engineMock binaryMessenger]) + .andReturn(binaryMessengerMock); + OCMStub([[engineMock ignoringNonObjectArgs] sendKeyEvent:FlutterKeyEvent {} + callback:nil + userData:nil]) + .andCall([FlutterViewControllerTestObjC class], + @selector(respondFalseForSendEvent:callback:userData:)); + FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engineMock + nibName:@"" + bundle:nil]; + id responderMock = flutter::testing::mockResponder(); + viewController.nextResponder = responderMock; + NSDictionary* expectedKeyDownEvent = @{ + @"keymap" : @"macos", + @"type" : @"keydown", + @"keyCode" : @(65), + @"modifiers" : @(538968064), + @"characters" : @".", + @"charactersIgnoringModifiers" : @".", + }; + NSData* encodedKeyDownEvent = + [[FlutterJSONMessageCodec sharedInstance] encode:expectedKeyDownEvent]; + NSDictionary* expectedKeyUpEvent = @{ + @"keymap" : @"macos", + @"type" : @"keyup", + @"keyCode" : @(65), + @"modifiers" : @(538968064), + @"characters" : @".", + @"charactersIgnoringModifiers" : @".", + }; + NSData* encodedKeyUpEvent = [[FlutterJSONMessageCodec sharedInstance] encode:expectedKeyUpEvent]; + CGEventRef cgEvent = CGEventCreateKeyboardEvent(NULL, 65, TRUE); + NSEvent* event = [NSEvent eventWithCGEvent:cgEvent]; + OCMExpect( // NOLINT(google-objc-avoid-throwing-exception) + [binaryMessengerMock sendOnChannel:@"flutter/keyevent" + message:encodedKeyDownEvent + binaryReply:[OCMArg any]]) + .andDo((^(NSInvocation* invocation) { + FlutterBinaryReply handler; + [invocation getArgument:&handler atIndex:4]; + NSDictionary* reply = @{ + @"handled" : @(true), + }; + NSData* encodedReply = [[FlutterJSONMessageCodec sharedInstance] encode:reply]; + handler(encodedReply); + })); + OCMExpect( // NOLINT(google-objc-avoid-throwing-exception) + [binaryMessengerMock sendOnChannel:@"flutter/keyevent" + message:encodedKeyUpEvent + binaryReply:[OCMArg any]]) + .andDo((^(NSInvocation* invocation) { + FlutterBinaryReply handler; + [invocation getArgument:&handler atIndex:4]; + NSDictionary* reply = @{ + @"handled" : @(true), + }; + NSData* encodedReply = [[FlutterJSONMessageCodec sharedInstance] encode:reply]; + handler(encodedReply); + })); + [viewController viewWillAppear]; // Initializes the event channel. + [viewController performKeyEquivalent:event]; + @try { + OCMVerify( // NOLINT(google-objc-avoid-throwing-exception) + [binaryMessengerMock sendOnChannel:@"flutter/keyevent" + message:encodedKeyDownEvent + binaryReply:[OCMArg any]]); + OCMVerify( // NOLINT(google-objc-avoid-throwing-exception) + [binaryMessengerMock sendOnChannel:@"flutter/keyevent" + message:encodedKeyUpEvent + binaryReply:[OCMArg any]]); + } @catch (...) { + return false; + } + return true; +} + + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event callback:(nullable FlutterKeyEventCallback)callback userData:(nullable void*)userData { diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h b/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h index a32f2f4f96764..f6dd076d112dd 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h @@ -5,6 +5,7 @@ #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterKeySecondaryResponder.h" +#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h" @interface FlutterViewController () @@ -17,6 +18,11 @@ */ @property(nonatomic, readonly, nonnull) NSPasteboard* pasteboard; +/** + * The text input plugin that handles text editing state for text fields. + */ +@property(nonatomic, readonly, nonnull) FlutterTextInputPlugin* textInputPlugin; + /** * Initializes this FlutterViewController with the specified `FlutterEngine`. * @@ -30,5 +36,9 @@ nibName:(nullable NSString*)nibName bundle:(nullable NSBundle*)nibBundle NS_DESIGNATED_INITIALIZER; -#pragma mark - Private interface declaration. +@end + +// Private methods made visible for testing +@interface FlutterViewController (TestMethods) +- (void)onAccessibilityStatusChanged:(nonnull NSNotification*)notification; @end diff --git a/third_party/accessibility/ax/platform/ax_platform_node.h b/third_party/accessibility/ax/platform/ax_platform_node.h index 08e5e5205d30e..295473dacfb9d 100644 --- a/third_party/accessibility/ax/platform/ax_platform_node.h +++ b/third_party/accessibility/ax/platform/ax_platform_node.h @@ -106,10 +106,10 @@ class AX_EXPORT AXPlatformNode { std::string SubtreeToString(); friend std::ostream& operator<<(std::ostream& stream, AXPlatformNode& node); + virtual ~AXPlatformNode(); protected: AXPlatformNode(); - virtual ~AXPlatformNode(); private: static std::vector ax_mode_observers_; diff --git a/third_party/accessibility/ax/platform/ax_platform_node_mac.mm b/third_party/accessibility/ax/platform/ax_platform_node_mac.mm index f891c71125740..78c318bba2e36 100644 --- a/third_party/accessibility/ax/platform/ax_platform_node_mac.mm +++ b/third_party/accessibility/ax/platform/ax_platform_node_mac.mm @@ -704,18 +704,13 @@ - (NSString*)AXSelectedTextInternal { } - (NSValue*)AXSelectedTextRangeInternal { - // Selection might not be supported. Return (NSRange){0,0} in that case. - int start = 0, end = 0; - if (_node->IsPlainTextField()) { - start = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelStart); - end = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelEnd); - } + int start = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelStart); + int end = _node->GetIntAttribute(ax::mojom::IntAttribute::kTextSelEnd); NSAssert((start >= 0 && end >= 0) || (start == -1 && end == -1), @"selection is invalid"); if (start == -1 && end == -1) { return [NSValue valueWithRange:{NSNotFound, 0}]; } - // NSRange cannot represent the direction the text was selected in. return [NSValue valueWithRange:{static_cast(std::min(start, end)), static_cast(abs(end - start))}];