diff --git a/BUILD.gn b/BUILD.gn index 9374834dc9137..5521ce8174325 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -55,6 +55,7 @@ group("flutter") { public_deps += [ "$flutter_root/flow:flow_unittests", "$flutter_root/fml:fml_unittests", + "$flutter_root/lib/ui:ui_unittests", "$flutter_root/runtime:runtime_unittests", "$flutter_root/shell/common:shell_unittests", "$flutter_root/shell/platform/common/cpp/client_wrapper:client_wrapper_unittests", diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 8d3693f1cc4cf..9b6ea9ff88dd7 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -316,6 +316,8 @@ FILE: ../../../flutter/lib/ui/dart_runtime_hooks.h FILE: ../../../flutter/lib/ui/dart_ui.cc FILE: ../../../flutter/lib/ui/dart_ui.h FILE: ../../../flutter/lib/ui/dart_wrapper.h +FILE: ../../../flutter/lib/ui/fixtures/DashInNooglerHat.jpg +FILE: ../../../flutter/lib/ui/fixtures/ui_test.dart FILE: ../../../flutter/lib/ui/geometry.dart FILE: ../../../flutter/lib/ui/hash_codes.dart FILE: ../../../flutter/lib/ui/hooks.dart @@ -342,6 +344,9 @@ FILE: ../../../flutter/lib/ui/painting/gradient.cc FILE: ../../../flutter/lib/ui/painting/gradient.h FILE: ../../../flutter/lib/ui/painting/image.cc FILE: ../../../flutter/lib/ui/painting/image.h +FILE: ../../../flutter/lib/ui/painting/image_decoder.cc +FILE: ../../../flutter/lib/ui/painting/image_decoder.h +FILE: ../../../flutter/lib/ui/painting/image_decoder_unittests.cc FILE: ../../../flutter/lib/ui/painting/image_encoding.cc FILE: ../../../flutter/lib/ui/painting/image_encoding.h FILE: ../../../flutter/lib/ui/painting/image_filter.cc @@ -350,6 +355,8 @@ FILE: ../../../flutter/lib/ui/painting/image_shader.cc FILE: ../../../flutter/lib/ui/painting/image_shader.h FILE: ../../../flutter/lib/ui/painting/matrix.cc FILE: ../../../flutter/lib/ui/painting/matrix.h +FILE: ../../../flutter/lib/ui/painting/multi_frame_codec.cc +FILE: ../../../flutter/lib/ui/painting/multi_frame_codec.h FILE: ../../../flutter/lib/ui/painting/paint.cc FILE: ../../../flutter/lib/ui/painting/paint.h FILE: ../../../flutter/lib/ui/painting/path.cc @@ -364,6 +371,8 @@ FILE: ../../../flutter/lib/ui/painting/rrect.cc FILE: ../../../flutter/lib/ui/painting/rrect.h FILE: ../../../flutter/lib/ui/painting/shader.cc FILE: ../../../flutter/lib/ui/painting/shader.h +FILE: ../../../flutter/lib/ui/painting/single_frame_codec.cc +FILE: ../../../flutter/lib/ui/painting/single_frame_codec.h FILE: ../../../flutter/lib/ui/painting/vertices.cc FILE: ../../../flutter/lib/ui/painting/vertices.h FILE: ../../../flutter/lib/ui/plugins.dart diff --git a/fml/concurrent_message_loop.cc b/fml/concurrent_message_loop.cc index 0c25db27619fe..482ceb479018d 100644 --- a/fml/concurrent_message_loop.cc +++ b/fml/concurrent_message_loop.cc @@ -11,10 +11,14 @@ namespace fml { -ConcurrentMessageLoop::ConcurrentMessageLoop() - : worker_count_(std::max(std::thread::hardware_concurrency(), 1u)), - shutdown_latch_(worker_count_), - shutdown_(false) { +std::shared_ptr ConcurrentMessageLoop::Create( + size_t worker_count) { + return std::shared_ptr{ + new ConcurrentMessageLoop(worker_count)}; +} + +ConcurrentMessageLoop::ConcurrentMessageLoop(size_t worker_count) + : worker_count_(std::max(worker_count, 1ul)) { for (size_t i = 0; i < worker_count_; ++i) { workers_.emplace_back([i, this]() { fml::Thread::SetCurrentThreadName( @@ -26,45 +30,97 @@ ConcurrentMessageLoop::ConcurrentMessageLoop() ConcurrentMessageLoop::~ConcurrentMessageLoop() { Terminate(); - shutdown_latch_.Wait(); for (auto& worker : workers_) { worker.join(); } } -// |fml::MessageLoopImpl| -void ConcurrentMessageLoop::Run() { - FML_CHECK(false); +size_t ConcurrentMessageLoop::GetWorkerCount() const { + return worker_count_; } -// |fml::MessageLoopImpl| -void ConcurrentMessageLoop::Terminate() { - std::scoped_lock lock(wait_condition_mutex_); - shutdown_ = true; - wait_condition_.notify_all(); +std::shared_ptr ConcurrentMessageLoop::GetTaskRunner() { + return std::make_shared(weak_from_this()); } -// |fml::MessageLoopImpl| -void ConcurrentMessageLoop::WakeUp(fml::TimePoint time_point) { - // Assume that the clocks are not the same. - const auto duration = std::chrono::nanoseconds( - (time_point - fml::TimePoint::Now()).ToNanoseconds()); - next_wake_ = std::chrono::high_resolution_clock::now() + duration; - wait_condition_.notify_all(); +void ConcurrentMessageLoop::PostTask(fml::closure task) { + if (!task) { + return; + } + + std::unique_lock lock(tasks_mutex_); + + // Don't just drop tasks on the floor in case of shutdown. + if (shutdown_) { + FML_DLOG(WARNING) + << "Tried to post a task to shutdown concurrent message " + "loop. The task will be executed on the callers thread."; + lock.unlock(); + task(); + return; + } + + tasks_.push(task); + + // Unlock the mutex before notifying the condition variable because that mutex + // has to be acquired on the other thread anyway. Waiting in this scope till + // it is acquired there is a pessimization. + lock.unlock(); + + tasks_condition_.notify_one(); } void ConcurrentMessageLoop::WorkerMain() { - while (!shutdown_) { - std::unique_lock lock(wait_condition_mutex_); - if (!shutdown_) { - wait_condition_.wait(lock); + while (true) { + std::unique_lock lock(tasks_mutex_); + tasks_condition_.wait(lock, + [&]() { return tasks_.size() > 0 || shutdown_; }); + + if (tasks_.size() == 0) { + // This can only be caused by shutdown. + FML_DCHECK(shutdown_); + break; } - TRACE_EVENT0("fml", "ConcurrentWorkerWake"); - RunSingleExpiredTaskNow(); + + auto task = tasks_.front(); + tasks_.pop(); + + // Don't hold onto the mutex while the task is being executed as it could + // itself try to post another tasks to this message loop. + lock.unlock(); + + TRACE_EVENT0("flutter", "ConcurrentWorkerWake"); + // Execute the one tasks we woke up for. + task(); + } +} + +void ConcurrentMessageLoop::Terminate() { + std::scoped_lock lock(tasks_mutex_); + shutdown_ = true; + tasks_condition_.notify_all(); +} + +ConcurrentTaskRunner::ConcurrentTaskRunner( + std::weak_ptr weak_loop) + : weak_loop_(std::move(weak_loop)) {} + +ConcurrentTaskRunner::~ConcurrentTaskRunner() = default; + +void ConcurrentTaskRunner::PostTask(fml::closure task) { + if (!task) { + return; + } + + if (auto loop = weak_loop_.lock()) { + loop->PostTask(task); + return; } - RunExpiredTasksNow(); - shutdown_latch_.CountDown(); + FML_DLOG(WARNING) + << "Tried to post to a concurrent message loop that has already died. " + "Executing the task on the callers thread."; + task(); } } // namespace fml diff --git a/fml/concurrent_message_loop.h b/fml/concurrent_message_loop.h index 7879d05239f83..0cfd85c90ec0f 100644 --- a/fml/concurrent_message_loop.h +++ b/fml/concurrent_message_loop.h @@ -5,51 +5,67 @@ #ifndef FLUTTER_FML_CONCURRENT_MESSAGE_LOOP_H_ #define FLUTTER_FML_CONCURRENT_MESSAGE_LOOP_H_ -#include -#include #include +#include #include -#include +#include "flutter/fml/closure.h" #include "flutter/fml/macros.h" -#include "flutter/fml/message_loop_impl.h" -#include "flutter/fml/synchronization/count_down_latch.h" #include "flutter/fml/synchronization/thread_annotations.h" namespace fml { -class ConcurrentMessageLoop : public MessageLoopImpl { - private: - const size_t worker_count_; - std::mutex wait_condition_mutex_; - std::condition_variable wait_condition_; - std::vector workers_; - CountDownLatch shutdown_latch_; - std::chrono::high_resolution_clock::time_point next_wake_; - std::atomic_bool shutdown_; +class ConcurrentTaskRunner; - ConcurrentMessageLoop(); +class ConcurrentMessageLoop + : public std::enable_shared_from_this { + public: + static std::shared_ptr Create( + size_t worker_count = std::thread::hardware_concurrency()); ~ConcurrentMessageLoop(); - // |fml::MessageLoopImpl| - void Run() override; + size_t GetWorkerCount() const; + + std::shared_ptr GetTaskRunner(); + + void Terminate(); - // |fml::MessageLoopImpl| - void Terminate() override; + private: + friend ConcurrentTaskRunner; - // |fml::MessageLoopImpl| - void WakeUp(fml::TimePoint time_point) override; + size_t worker_count_ = 0; + std::vector workers_; + std::mutex tasks_mutex_; + std::condition_variable tasks_condition_; + std::queue tasks_; + bool shutdown_ = false; - static void WorkerMain(ConcurrentMessageLoop* loop); + ConcurrentMessageLoop(size_t worker_count); void WorkerMain(); - FML_FRIEND_MAKE_REF_COUNTED(ConcurrentMessageLoop); - FML_FRIEND_REF_COUNTED_THREAD_SAFE(ConcurrentMessageLoop); + void PostTask(fml::closure task); + FML_DISALLOW_COPY_AND_ASSIGN(ConcurrentMessageLoop); }; +class ConcurrentTaskRunner { + public: + ConcurrentTaskRunner(std::weak_ptr weak_loop); + + ~ConcurrentTaskRunner(); + + void PostTask(fml::closure task); + + private: + friend ConcurrentMessageLoop; + + std::weak_ptr weak_loop_; + + FML_DISALLOW_COPY_AND_ASSIGN(ConcurrentTaskRunner); +}; + } // namespace fml #endif // FLUTTER_FML_CONCURRENT_MESSAGE_LOOP_H_ diff --git a/fml/message_loop.cc b/fml/message_loop.cc index bface19f9e17f..1bf1acd2498c0 100644 --- a/fml/message_loop.cc +++ b/fml/message_loop.cc @@ -6,7 +6,6 @@ #include -#include "flutter/fml/concurrent_message_loop.h" #include "flutter/fml/memory/ref_counted.h" #include "flutter/fml/memory/ref_ptr.h" #include "flutter/fml/message_loop_impl.h" @@ -44,13 +43,6 @@ MessageLoop::MessageLoop() FML_CHECK(task_runner_); } -MessageLoop::MessageLoop(Type) - : loop_(fml::MakeRefCounted()), - task_runner_(fml::MakeRefCounted(loop_)) { - FML_CHECK(loop_); - FML_CHECK(task_runner_); -} - MessageLoop::~MessageLoop() = default; void MessageLoop::Run() { diff --git a/fml/message_loop.h b/fml/message_loop.h index 30db680cb9dd3..961f92bce230a 100644 --- a/fml/message_loop.h +++ b/fml/message_loop.h @@ -18,10 +18,6 @@ class MessageLoop { FML_EMBEDDER_ONLY static MessageLoop& GetCurrent(); - enum class Type { kConcurrent }; - - MessageLoop(Type type); - bool IsValid() const; void Run(); diff --git a/fml/message_loop_unittests.cc b/fml/message_loop_unittests.cc index d7c0a9cb926e9..94585d22cee80 100644 --- a/fml/message_loop_unittests.cc +++ b/fml/message_loop_unittests.cc @@ -7,6 +7,7 @@ #include #include +#include "flutter/fml/concurrent_message_loop.h" #include "flutter/fml/message_loop.h" #include "flutter/fml/synchronization/count_down_latch.h" #include "flutter/fml/synchronization/waitable_event.h" @@ -281,19 +282,31 @@ TEST(MessageLoop, TaskObserverFire) { ASSERT_TRUE(terminated); } +TEST(MessageLoop, CanCreateAndShutdownConcurrentMessageLoopsOverAndOver) { + for (size_t i = 0; i < 10; ++i) { + auto loop = fml::ConcurrentMessageLoop::Create(i + 1); + ASSERT_EQ(loop->GetWorkerCount(), i + 1); + } +} + TEST(MessageLoop, CanCreateConcurrentMessageLoop) { - fml::MessageLoop loop(fml::MessageLoop::Type::kConcurrent); - auto task_runner = loop.GetTaskRunner(); + auto loop = fml::ConcurrentMessageLoop::Create(); + auto task_runner = loop->GetTaskRunner(); const size_t kCount = 10; fml::CountDownLatch latch(kCount); + std::mutex thread_ids_mutex; + std::set thread_ids; for (size_t i = 0; i < kCount; ++i) { - task_runner->PostTask([&latch]() { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); + task_runner->PostTask([&]() { + std::this_thread::sleep_for(std::chrono::seconds(1)); std::cout << "Ran on thread: " << std::this_thread::get_id() << std::endl; + std::scoped_lock lock(thread_ids_mutex); + thread_ids.insert(std::this_thread::get_id()); latch.CountDown(); }); } latch.Wait(); + ASSERT_GE(thread_ids.size(), 1u); } TEST(MessageLoop, CanSwapMessageLoopsAndPreserveThreadConfiguration) { diff --git a/fml/trace_event.h b/fml/trace_event.h index 9f86bcfa04b0c..d37b581f333af 100644 --- a/fml/trace_event.h +++ b/fml/trace_event.h @@ -244,6 +244,41 @@ class ScopedInstantEnd { FML_DISALLOW_COPY_AND_ASSIGN(ScopedInstantEnd); }; +// A move-only utility object that creates a new flow with a unique ID and +// automatically ends it when it goes out of scope. When tracing using multiple +// overlapping flows, it often gets hard to make sure to end the flow +// (especially with early returns), or, end/step on the wrong flow. This +// leads to corrupted or missing traces in the UI. +class TraceFlow { + public: + TraceFlow(const char* label) : label_(label), nonce_(TraceNonce()) { + TraceEventFlowBegin0("flutter", label_, nonce_); + } + + ~TraceFlow() { End(label_); } + + TraceFlow(TraceFlow&& other) : label_(other.label_), nonce_(other.nonce_) { + other.nonce_ = 0; + } + + void Step(const char* label) const { + TraceEventFlowStep0("flutter", label, nonce_); + } + + void End(const char* label = nullptr) { + if (nonce_ != 0) { + TraceEventFlowEnd0("flutter", label == nullptr ? label_ : label, nonce_); + nonce_ = 0; + } + } + + private: + const char* label_; + size_t nonce_; + + FML_DISALLOW_COPY_AND_ASSIGN(TraceFlow); +}; + } // namespace tracing } // namespace fml diff --git a/lib/ui/BUILD.gn b/lib/ui/BUILD.gn index e7cc9a66bdc4d..06f1aa786936b 100644 --- a/lib/ui/BUILD.gn +++ b/lib/ui/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//build/fuchsia/sdk.gni") +import("$flutter_root/testing/testing.gni") source_set("ui") { sources = [ @@ -34,6 +35,8 @@ source_set("ui") { "painting/gradient.h", "painting/image.cc", "painting/image.h", + "painting/image_decoder.cc", + "painting/image_decoder.h", "painting/image_encoding.cc", "painting/image_encoding.h", "painting/image_filter.cc", @@ -42,6 +45,8 @@ source_set("ui") { "painting/image_shader.h", "painting/matrix.cc", "painting/matrix.h", + "painting/multi_frame_codec.cc", + "painting/multi_frame_codec.h", "painting/paint.cc", "painting/paint.h", "painting/path.cc", @@ -56,6 +61,8 @@ source_set("ui") { "painting/rrect.h", "painting/shader.cc", "painting/shader.h", + "painting/single_frame_codec.cc", + "painting/single_frame_codec.h", "painting/vertices.cc", "painting/vertices.h", "plugins/callback_cache.cc", @@ -136,3 +143,26 @@ source_set("ui") { } } } + +if (current_toolchain == host_toolchain) { + test_fixtures("ui_unittests_fixtures") { + dart_main = "fixtures/ui_test.dart" + fixtures = [ "fixtures/DashInNooglerHat.jpg" ] + } + + executable("ui_unittests") { + testonly = true + + sources = [ + "painting/image_decoder_unittests.cc", + ] + + deps = [ + ":ui", + ":ui_unittests_fixtures", + "$flutter_root/common", + "$flutter_root/testing:dart", + "$flutter_root/testing:opengl", + ] + } +} diff --git a/lib/ui/fixtures/DashInNooglerHat.jpg b/lib/ui/fixtures/DashInNooglerHat.jpg new file mode 100644 index 0000000000000..488fdb4d5215c Binary files /dev/null and b/lib/ui/fixtures/DashInNooglerHat.jpg differ diff --git a/lib/ui/fixtures/ui_test.dart b/lib/ui/fixtures/ui_test.dart new file mode 100644 index 0000000000000..f9b0dd79fed95 --- /dev/null +++ b/lib/ui/fixtures/ui_test.dart @@ -0,0 +1,5 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +void main() {} diff --git a/lib/ui/io_manager.h b/lib/ui/io_manager.h index 28cf4f908806e..200d732f115ad 100644 --- a/lib/ui/io_manager.h +++ b/lib/ui/io_manager.h @@ -17,6 +17,8 @@ class IOManager { public: virtual ~IOManager() = default; + virtual fml::WeakPtr GetWeakIOManager() const = 0; + virtual fml::WeakPtr GetResourceContext() const = 0; virtual fml::RefPtr GetSkiaUnrefQueue() const = 0; diff --git a/lib/ui/painting/codec.cc b/lib/ui/painting/codec.cc index d8418bd7455ca..b181837e3f51f 100644 --- a/lib/ui/painting/codec.cc +++ b/lib/ui/painting/codec.cc @@ -4,11 +4,15 @@ #include "flutter/lib/ui/painting/codec.h" +#include + #include "flutter/common/task_runners.h" #include "flutter/fml/logging.h" #include "flutter/fml/make_copyable.h" #include "flutter/fml/trace_event.h" #include "flutter/lib/ui/painting/frame_info.h" +#include "flutter/lib/ui/painting/multi_frame_codec.h" +#include "flutter/lib/ui/painting/single_frame_codec.h" #include "third_party/skia/include/codec/SkCodec.h" #include "third_party/skia/include/core/SkPixelRef.h" #include "third_party/tonic/dart_binding_macros.h" @@ -23,11 +27,6 @@ using tonic::ToDart; namespace flutter { -namespace { - -static constexpr const char* kInitCodecTraceTag = "InitCodec"; -static constexpr const char* kCodecNextFrameTraceTag = "CodecNextFrame"; - // This needs to be kept in sync with _kDoNotResizeDimension in painting.dart const int kDoNotResizeDimension = -1; @@ -37,284 +36,27 @@ enum PixelFormat { kBGRA8888, }; -struct ImageInfo { - SkImageInfo sk_info; - size_t row_bytes; -}; - -static void InvokeCodecCallback(fml::RefPtr codec, - std::unique_ptr callback, - size_t trace_id) { - std::shared_ptr dart_state = callback->dart_state().lock(); - if (!dart_state) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); - return; - } - tonic::DartState::Scope scope(dart_state); - if (!codec) { - DartInvoke(callback->value(), {Dart_Null()}); - } else { - DartInvoke(callback->value(), {ToDart(codec)}); - } - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); -} - -static sk_sp DecodeImage(fml::WeakPtr context, - sk_sp buffer, - size_t trace_id) { - TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id); - TRACE_EVENT0("flutter", "DecodeImage"); - - if (buffer == nullptr || buffer->isEmpty()) { - return nullptr; - } - - if (context) { - // This indicates that we do not want a "linear blending" decode. - sk_sp dstColorSpace = nullptr; - return SkImage::MakeCrossContextFromEncoded( - context.get(), std::move(buffer), true, dstColorSpace.get(), true); - } else { - // Defer decoding until time of draw later on the GPU thread. Can happen - // when GL operations are currently forbidden such as in the background - // on iOS. - return SkImage::MakeFromEncoded(std::move(buffer)); - } -} - -// Returns true if the image needs to be resized. -// -// newWidth and newHeight will reflect the dimensions that the image should -// be scaled to. -// -// The targetWidth and targetHeight arguments specify the size of the output -// image, in image pixels. If they are not equal to the intrinsic dimensions of -// the image, then the image will be scaled after being decoded. If exactly one -// of these two arguments is equal to kDoNotResizeDimension, then the aspect -// ratio will be maintained while forcing the image to match the other given -// dimension. If both are equal to kDoNotResizeDimension, then the image -// maintains its real size. -static bool needsResize(const int currentWidth, - const int currentHeight, - const int targetWidth, - const int targetHeight, - int& newWidth, - int& newHeight) { - newWidth = currentWidth; - newHeight = currentHeight; - if (targetWidth == kDoNotResizeDimension && - targetHeight == kDoNotResizeDimension) { - return false; - } - - if (currentWidth == targetWidth && currentHeight == targetHeight) { - return false; - } - - if (targetWidth == kDoNotResizeDimension) { - newHeight = targetHeight; - const double aspectRatio = (double)currentWidth / currentHeight; - newWidth = round(aspectRatio * newHeight); - return true; - } else if (targetHeight == kDoNotResizeDimension) { - newWidth = targetWidth; - const double invAspectRatio = (double)currentHeight / currentWidth; - newHeight = round(invAspectRatio * newWidth); - return true; - } else { - newWidth = targetWidth; - newHeight = targetHeight; - return true; - } -} - -static sk_sp ResizeImageToExactSize(fml::WeakPtr context, - sk_sp image, - SkImageInfo scaledImageInfo) { - if (image == nullptr || !image.get()) { - FML_LOG(ERROR) << "Failed to decode image."; - return nullptr; - } - - SkBitmap bitmap = SkBitmap(); - if (!bitmap.tryAllocPixels(scaledImageInfo)) { - FML_LOG(ERROR) << "Failed to allocate bitmap."; - return nullptr; - } - - if (!image->scalePixels(bitmap.pixmap(), kLow_SkFilterQuality)) { - FML_LOG(ERROR) << "Failed to scale pixels."; - return nullptr; - } - - // This indicates that we do not want a "linear blending" decode. - sk_sp dstColorSpace = nullptr; - GrContext* grContext = context ? context.get() : nullptr; - return SkImage::MakeCrossContextFromPixmap(grContext, bitmap.pixmap(), true, - dstColorSpace.get(), true); -} - -static sk_sp DecodeAndResizeImageToExactSize( - fml::WeakPtr context, - SkImageInfo scaledImageInfo, - sk_sp buffer, - size_t trace_id) { - TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id); - TRACE_EVENT0("flutter", "DecodeAndResizeImageToExactSize"); - - // Do not create a cross context image here, since it can not be resized. - sk_sp image = SkImage::MakeFromEncoded(std::move(buffer)); - return ResizeImageToExactSize(context, image, scaledImageInfo); -} - -static sk_sp DecodeAndResizeImage(fml::WeakPtr context, - std::unique_ptr& skCodec, - sk_sp buffer, - const int targetWidth, - const int targetHeight, - size_t trace_id) { - const SkImageInfo imageInfo = skCodec->getInfo(); - - const int width = imageInfo.width(); - const int height = imageInfo.height(); - - int newWidth, newHeight; - if (needsResize(width, height, targetWidth, targetHeight, newWidth, - newHeight)) { - return DecodeAndResizeImageToExactSize( - context, imageInfo.makeWH(newWidth, newHeight), buffer, trace_id); - } else { - return DecodeImage(context, buffer, trace_id); - } -} - -fml::RefPtr InitCodec(fml::WeakPtr context, - sk_sp buffer, - fml::RefPtr unref_queue, - const int targetWidth, - const int targetHeight, - size_t trace_id) { - TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id); - TRACE_EVENT0("blink", "InitCodec"); - - if (buffer == nullptr || buffer->isEmpty()) { - FML_LOG(ERROR) << "InitCodec failed - buffer was empty "; - return nullptr; - } - - std::unique_ptr skCodec = SkCodec::MakeFromData(buffer); - if (!skCodec) { - FML_LOG(ERROR) << "Failed decoding image. Data is either invalid, or it is " - "encoded using an unsupported format."; - return nullptr; - } - if (skCodec->getFrameCount() > 1) { - return fml::MakeRefCounted(std::move(skCodec)); - } - - auto skImage = DecodeAndResizeImage(context, skCodec, buffer, targetWidth, - targetHeight, trace_id); - FML_DCHECK(skImage) << "Unable to resize the image to (w, h): " << targetWidth - << ", " << targetHeight << "."; - if (!skImage) { - return nullptr; - } - auto image = CanvasImage::Create(); - image->set_image({skImage, unref_queue}); - auto frameInfo = fml::MakeRefCounted(std::move(image), 0); - return fml::MakeRefCounted(std::move(frameInfo)); -} - -fml::RefPtr InitCodecUncompressed( - fml::WeakPtr context, - sk_sp buffer, - ImageInfo image_info, - fml::RefPtr unref_queue, - int targetWidth, - int targetHeight, - size_t trace_id) { - TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id); - TRACE_EVENT0("blink", "InitCodecUncompressed"); - - if (buffer == nullptr || buffer->isEmpty()) { - FML_LOG(ERROR) << "InitCodecUncompressed failed - buffer was empty"; - return nullptr; - } - - sk_sp skImage; - int newWidth, newHeight; - if (needsResize(image_info.sk_info.width(), image_info.sk_info.height(), - targetWidth, targetHeight, newWidth, newHeight)) { - auto imageToResize = SkImage::MakeRasterData( - image_info.sk_info, std::move(buffer), image_info.row_bytes); - skImage = ResizeImageToExactSize( - context, imageToResize, image_info.sk_info.makeWH(newWidth, newHeight)); - } else if (context) { - SkPixmap pixmap(image_info.sk_info, buffer->data(), image_info.row_bytes); - skImage = SkImage::MakeCrossContextFromPixmap(context.get(), pixmap, true, - nullptr, true); - } else { - skImage = SkImage::MakeRasterData(image_info.sk_info, std::move(buffer), - image_info.row_bytes); - } - - auto image = CanvasImage::Create(); - image->set_image({skImage, unref_queue}); - auto frameInfo = fml::MakeRefCounted(std::move(image), 0); - return fml::MakeRefCounted(std::move(frameInfo)); -} - -void InitCodecAndInvokeCodecCallback( - fml::RefPtr ui_task_runner, - fml::WeakPtr context, - fml::RefPtr unref_queue, - std::unique_ptr callback, - sk_sp buffer, - std::unique_ptr image_info, - const int targetWidth, - const int targetHeight, - size_t trace_id) { - fml::RefPtr codec; - if (image_info) { - codec = InitCodecUncompressed(context, std::move(buffer), *image_info, - std::move(unref_queue), targetWidth, - targetHeight, trace_id); - } else { - codec = InitCodec(context, std::move(buffer), std::move(unref_queue), - targetWidth, targetHeight, trace_id); - } - ui_task_runner->PostTask( - fml::MakeCopyable([callback = std::move(callback), - codec = std::move(codec), trace_id]() mutable { - InvokeCodecCallback(std::move(codec), std::move(callback), trace_id); - })); -} - -bool ConvertImageInfo(Dart_Handle image_info_handle, - Dart_NativeArguments args, - ImageInfo* image_info) { +static std::variant ConvertImageInfo( + Dart_Handle image_info_handle, + Dart_NativeArguments args) { Dart_Handle width_handle = Dart_GetField(image_info_handle, ToDart("width")); if (!Dart_IsInteger(width_handle)) { - Dart_SetReturnValue(args, ToDart("ImageInfo.width must be an integer")); - return false; + return "ImageInfo.width must be an integer"; } Dart_Handle height_handle = Dart_GetField(image_info_handle, ToDart("height")); if (!Dart_IsInteger(height_handle)) { - Dart_SetReturnValue(args, ToDart("ImageInfo.height must be an integer")); - return false; + return "ImageInfo.height must be an integer"; } Dart_Handle format_handle = Dart_GetField(image_info_handle, ToDart("format")); if (!Dart_IsInteger(format_handle)) { - Dart_SetReturnValue(args, ToDart("ImageInfo.format must be an integer")); - return false; + return "ImageInfo.format must be an integer"; } Dart_Handle row_bytes_handle = Dart_GetField(image_info_handle, ToDart("rowBytes")); if (!Dart_IsInteger(row_bytes_handle)) { - Dart_SetReturnValue(args, ToDart("ImageInfo.rowBytes must be an integer")); - return false; + return "ImageInfo.rowBytes must be an integer"; } PixelFormat pixel_format = static_cast( @@ -329,70 +71,71 @@ bool ConvertImageInfo(Dart_Handle image_info_handle, break; } if (color_type == kUnknown_SkColorType) { - Dart_SetReturnValue(args, ToDart("Invalid pixel format")); - return false; + return "Invalid pixel format"; } int width = tonic::DartConverter::FromDart(width_handle); if (width <= 0) { - Dart_SetReturnValue(args, ToDart("width must be greater than zero")); - return false; + return "width must be greater than zero"; } int height = tonic::DartConverter::FromDart(height_handle); if (height <= 0) { - Dart_SetReturnValue(args, ToDart("height must be greater than zero")); - return false; + return "height must be greater than zero"; } - image_info->sk_info = + + ImageDecoder::ImageInfo image_info; + image_info.sk_info = SkImageInfo::Make(width, height, color_type, kPremul_SkAlphaType); - image_info->row_bytes = + image_info.row_bytes = tonic::DartConverter::FromDart(row_bytes_handle); - if (image_info->row_bytes < image_info->sk_info.minRowBytes()) { - Dart_SetReturnValue( - args, ToDart("rowBytes does not match the width of the image")); - return false; + if (image_info.row_bytes < image_info.sk_info.minRowBytes()) { + return "rowBytes does not match the width of the image"; } - return true; + return image_info; } -void InstantiateImageCodec(Dart_NativeArguments args) { - static size_t trace_counter = 1; - const size_t trace_id = trace_counter++; - TRACE_FLOW_BEGIN("flutter", kInitCodecTraceTag, trace_id); - +static void InstantiateImageCodec(Dart_NativeArguments args) { Dart_Handle callback_handle = Dart_GetNativeArgument(args, 1); if (!Dart_IsClosure(callback_handle)) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); - Dart_SetReturnValue(args, ToDart("Callback must be a function")); + Dart_SetReturnValue(args, tonic::ToDart("Callback must be a function")); return; } Dart_Handle image_info_handle = Dart_GetNativeArgument(args, 2); - std::unique_ptr image_info; + + std::optional image_info; + if (!Dart_IsNull(image_info_handle)) { - image_info = std::make_unique(); - if (!ConvertImageInfo(image_info_handle, args, image_info.get())) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); + auto image_info_results = ConvertImageInfo(image_info_handle, args); + if (auto value = + std::get_if(&image_info_results)) { + image_info = *value; + } else if (auto error = std::get_if(&image_info_results)) { + Dart_SetReturnValue(args, tonic::ToDart(*error)); return; } } - Dart_Handle exception = nullptr; - tonic::Uint8List list = - tonic::DartConverter::FromArguments(args, 0, exception); - if (exception) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); - Dart_SetReturnValue(args, exception); - return; + sk_sp buffer; + + { + Dart_Handle exception = nullptr; + tonic::Uint8List list = + tonic::DartConverter::FromArguments(args, 0, + exception); + if (exception) { + Dart_SetReturnValue(args, exception); + return; + } + buffer = SkData::MakeWithCopy(list.data(), list.num_elements()); } if (image_info) { - int expected_size = image_info->row_bytes * image_info->sk_info.height(); - if (list.num_elements() < expected_size) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); - list.Release(); + const auto expected_size = + image_info->row_bytes * image_info->sk_info.height(); + if (buffer->size() < expected_size) { Dart_SetReturnValue( args, ToDart("Pixel buffer size does not match image size")); return; @@ -404,73 +147,34 @@ void InstantiateImageCodec(Dart_NativeArguments args) { const int targetHeight = tonic::DartConverter::FromDart(Dart_GetNativeArgument(args, 4)); - auto buffer = SkData::MakeWithCopy(list.data(), list.num_elements()); - - auto* dart_state = UIDartState::Current(); - - const auto& task_runners = dart_state->GetTaskRunners(); - task_runners.GetIOTaskRunner()->PostTask(fml::MakeCopyable( - [callback = std::make_unique( - tonic::DartState::Current(), callback_handle), - buffer = std::move(buffer), trace_id, image_info = std::move(image_info), - ui_task_runner = task_runners.GetUITaskRunner(), - context = dart_state->GetResourceContext(), - queue = UIDartState::Current()->GetSkiaUnrefQueue(), targetWidth, - targetHeight]() mutable { - InitCodecAndInvokeCodecCallback( - std::move(ui_task_runner), context, std::move(queue), - std::move(callback), std::move(buffer), std::move(image_info), - targetWidth, targetHeight, trace_id); - })); -} + auto codec = SkCodec::MakeFromData(buffer); -bool copy_to(SkBitmap* dst, SkColorType dstColorType, const SkBitmap& src) { - SkPixmap srcPM; - if (!src.peekPixels(&srcPM)) { - return false; - } - - SkBitmap tmpDst; - SkImageInfo dstInfo = srcPM.info().makeColorType(dstColorType); - if (!tmpDst.setInfo(dstInfo)) { - return false; - } - - if (!tmpDst.tryAllocPixels()) { - return false; + if (!codec) { + Dart_SetReturnValue(args, ToDart("Could not instantiate image codec.")); + return; } - SkPixmap dstPM; - if (!tmpDst.peekPixels(&dstPM)) { - return false; - } + fml::RefPtr ui_codec; - if (!srcPM.readPixels(dstPM)) { - return false; - } + if (codec->getFrameCount() == 1) { + ImageDecoder::ImageDescriptor descriptor; + descriptor.decompressed_image_info = image_info; - dst->swap(tmpDst); - return true; -} + if (targetWidth != kDoNotResizeDimension) { + descriptor.target_width = targetWidth; + } + if (targetHeight != kDoNotResizeDimension) { + descriptor.target_height = targetHeight; + } + descriptor.data = std::move(buffer); -void InvokeNextFrameCallback(fml::RefPtr frameInfo, - std::unique_ptr callback, - size_t trace_id) { - std::shared_ptr dart_state = callback->dart_state().lock(); - if (!dart_state) { - TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id); - return; - } - tonic::DartState::Scope scope(dart_state); - if (!frameInfo) { - DartInvoke(callback->value(), {Dart_Null()}); + ui_codec = fml::MakeRefCounted(std::move(descriptor)); } else { - DartInvoke(callback->value(), {ToDart(frameInfo)}); + ui_codec = fml::MakeRefCounted(std::move(codec)); } - TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id); -} -} // namespace + tonic::DartInvoke(callback_handle, {ToDart(ui_codec)}); +} IMPLEMENT_WRAPPERTYPEINFO(ui, Codec); @@ -486,166 +190,6 @@ void Codec::dispose() { ClearDartWrapper(); } -MultiFrameCodec::MultiFrameCodec(std::unique_ptr codec) - : codec_(std::move(codec)), - frameCount_(codec_->getFrameCount()), - repetitionCount_(codec_->getRepetitionCount()), - nextFrameIndex_(0) {} - -MultiFrameCodec::~MultiFrameCodec() {} - -int MultiFrameCodec::frameCount() const { - return frameCount_; -} - -int MultiFrameCodec::repetitionCount() const { - return repetitionCount_; -} - -sk_sp MultiFrameCodec::GetNextFrameImage( - fml::WeakPtr resourceContext) { - SkBitmap bitmap = SkBitmap(); - SkImageInfo info = codec_->getInfo().makeColorType(kN32_SkColorType); - if (info.alphaType() == kUnpremul_SkAlphaType) { - info = info.makeAlphaType(kPremul_SkAlphaType); - } - bitmap.allocPixels(info); - - SkCodec::Options options; - options.fFrameIndex = nextFrameIndex_; - SkCodec::FrameInfo frameInfo; - codec_->getFrameInfo(nextFrameIndex_, &frameInfo); - const int requiredFrameIndex = frameInfo.fRequiredFrame; - if (requiredFrameIndex != SkCodec::kNoFrame) { - if (lastRequiredFrame_ == nullptr) { - FML_LOG(ERROR) << "Frame " << nextFrameIndex_ << " depends on frame " - << requiredFrameIndex - << " and no required frames are cached."; - return NULL; - } else if (lastRequiredFrameIndex_ != requiredFrameIndex) { - FML_DLOG(INFO) << "Required frame " << requiredFrameIndex - << " is not cached. Using " << lastRequiredFrameIndex_ - << " instead"; - } - - if (lastRequiredFrame_->getPixels() && - copy_to(&bitmap, lastRequiredFrame_->colorType(), - *lastRequiredFrame_)) { - options.fPriorFrame = requiredFrameIndex; - } - } - - if (SkCodec::kSuccess != codec_->getPixels(info, bitmap.getPixels(), - bitmap.rowBytes(), &options)) { - FML_LOG(ERROR) << "Could not getPixels for frame " << nextFrameIndex_; - return NULL; - } - - // Hold onto this if we need it to decode future frames. - if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kKeep) { - lastRequiredFrame_ = std::make_unique(bitmap); - lastRequiredFrameIndex_ = nextFrameIndex_; - } - - if (resourceContext) { - SkPixmap pixmap(bitmap.info(), bitmap.pixelRef()->pixels(), - bitmap.pixelRef()->rowBytes()); - // This indicates that we do not want a "linear blending" decode. - sk_sp dstColorSpace = nullptr; - return SkImage::MakeCrossContextFromPixmap(resourceContext.get(), pixmap, - true, dstColorSpace.get()); - } else { - // Defer decoding until time of draw later on the GPU thread. Can happen - // when GL operations are currently forbidden such as in the background - // on iOS. - return SkImage::MakeFromBitmap(bitmap); - } -} - -void MultiFrameCodec::GetNextFrameAndInvokeCallback( - std::unique_ptr callback, - fml::RefPtr ui_task_runner, - fml::WeakPtr resourceContext, - fml::RefPtr unref_queue, - size_t trace_id) { - fml::RefPtr frameInfo = NULL; - sk_sp skImage = GetNextFrameImage(resourceContext); - if (skImage) { - fml::RefPtr image = CanvasImage::Create(); - image->set_image({skImage, std::move(unref_queue)}); - SkCodec::FrameInfo skFrameInfo; - codec_->getFrameInfo(nextFrameIndex_, &skFrameInfo); - frameInfo = - fml::MakeRefCounted(std::move(image), skFrameInfo.fDuration); - } - nextFrameIndex_ = (nextFrameIndex_ + 1) % frameCount_; - - ui_task_runner->PostTask(fml::MakeCopyable( - [callback = std::move(callback), frameInfo, trace_id]() mutable { - InvokeNextFrameCallback(frameInfo, std::move(callback), trace_id); - })); - - TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id); -} - -Dart_Handle MultiFrameCodec::getNextFrame(Dart_Handle callback_handle) { - static size_t trace_counter = 1; - const size_t trace_id = trace_counter++; - TRACE_FLOW_BEGIN("flutter", kCodecNextFrameTraceTag, trace_id); - - if (!Dart_IsClosure(callback_handle)) { - TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id); - return ToDart("Callback must be a function"); - } - - auto* dart_state = UIDartState::Current(); - - const auto& task_runners = dart_state->GetTaskRunners(); - - task_runners.GetIOTaskRunner()->PostTask(fml::MakeCopyable( - [callback = std::make_unique( - tonic::DartState::Current(), callback_handle), - this, trace_id, ui_task_runner = task_runners.GetUITaskRunner(), - queue = UIDartState::Current()->GetSkiaUnrefQueue(), - context = dart_state->GetResourceContext()]() mutable { - GetNextFrameAndInvokeCallback(std::move(callback), - std::move(ui_task_runner), context, - std::move(queue), trace_id); - })); - - return Dart_Null(); -} - -SingleFrameCodec::SingleFrameCodec(fml::RefPtr frame) - : frame_(std::move(frame)) {} - -SingleFrameCodec::~SingleFrameCodec() {} - -int SingleFrameCodec::frameCount() const { - return 1; -} - -int SingleFrameCodec::repetitionCount() const { - return 0; -} - -Dart_Handle SingleFrameCodec::getNextFrame(Dart_Handle callback_handle) { - if (!Dart_IsClosure(callback_handle)) { - return ToDart("Callback must be a function"); - } - - auto callback = std::make_unique( - tonic::DartState::Current(), callback_handle); - std::shared_ptr dart_state = callback->dart_state().lock(); - if (!dart_state) { - return ToDart("Invalid dart state"); - } - - tonic::DartState::Scope scope(dart_state); - DartInvoke(callback->value(), {ToDart(frame_)}); - return Dart_Null(); -} - void Codec::RegisterNatives(tonic::DartLibraryNatives* natives) { natives->Register({ {"instantiateImageCodec", InstantiateImageCodec, 5, true}, diff --git a/lib/ui/painting/codec.h b/lib/ui/painting/codec.h index 776e97cc58c53..2e0c746eac2d6 100644 --- a/lib/ui/painting/codec.h +++ b/lib/ui/painting/codec.h @@ -27,63 +27,16 @@ class Codec : public RefCountedDartWrappable { public: virtual int frameCount() const = 0; + virtual int repetitionCount() const = 0; + virtual Dart_Handle getNextFrame(Dart_Handle callback_handle) = 0; + void dispose(); static void RegisterNatives(tonic::DartLibraryNatives* natives); }; -class MultiFrameCodec : public Codec { - public: - int frameCount() const override; - int repetitionCount() const override; - Dart_Handle getNextFrame(Dart_Handle args) override; - - private: - MultiFrameCodec(std::unique_ptr codec); - - ~MultiFrameCodec() override; - - sk_sp GetNextFrameImage(fml::WeakPtr resourceContext); - - void GetNextFrameAndInvokeCallback( - std::unique_ptr callback, - fml::RefPtr ui_task_runner, - fml::WeakPtr resourceContext, - fml::RefPtr unref_queue, - size_t trace_id); - - const std::unique_ptr codec_; - const int frameCount_; - const int repetitionCount_; - int nextFrameIndex_; - - // The last decoded frame that's required to decode any subsequent frames. - std::unique_ptr lastRequiredFrame_; - // The index of the last decoded required frame. - int lastRequiredFrameIndex_ = -1; - - FML_FRIEND_MAKE_REF_COUNTED(MultiFrameCodec); - FML_FRIEND_REF_COUNTED_THREAD_SAFE(MultiFrameCodec); -}; - -class SingleFrameCodec : public Codec { - public: - int frameCount() const override; - int repetitionCount() const override; - Dart_Handle getNextFrame(Dart_Handle args) override; - - private: - SingleFrameCodec(fml::RefPtr frame); - ~SingleFrameCodec() override; - - fml::RefPtr frame_; - - FML_FRIEND_MAKE_REF_COUNTED(SingleFrameCodec); - FML_FRIEND_REF_COUNTED_THREAD_SAFE(SingleFrameCodec); -}; - } // namespace flutter #endif // FLUTTER_LIB_UI_PAINTING_CODEC_H_ diff --git a/lib/ui/painting/frame_info.h b/lib/ui/painting/frame_info.h index 2637d3e66d182..184b132d17791 100644 --- a/lib/ui/painting/frame_info.h +++ b/lib/ui/painting/frame_info.h @@ -8,10 +8,6 @@ #include "flutter/lib/ui/dart_wrapper.h" #include "flutter/lib/ui/painting/image.h" -namespace tonic { -class DartLibraryNatives; -} // namespace tonic - namespace flutter { // A single animation frame. diff --git a/lib/ui/painting/image.cc b/lib/ui/painting/image.cc index a0a4bfcb02737..8ee65790924bf 100644 --- a/lib/ui/painting/image.cc +++ b/lib/ui/painting/image.cc @@ -42,7 +42,10 @@ void CanvasImage::dispose() { size_t CanvasImage::GetAllocationSize() { if (auto image = image_.get()) { - return image->width() * image->height() * 4; + const auto& info = image->imageInfo(); + const auto kMipmapOverhead = 4.0 / 3.0; + const size_t image_byte_size = info.computeMinByteSize() * kMipmapOverhead; + return image_byte_size + sizeof(this); } else { return sizeof(CanvasImage); } diff --git a/lib/ui/painting/image_decoder.cc b/lib/ui/painting/image_decoder.cc new file mode 100644 index 0000000000000..200349eb4b636 --- /dev/null +++ b/lib/ui/painting/image_decoder.cc @@ -0,0 +1,336 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/ui/painting/image_decoder.h" + +#include "flutter/fml/make_copyable.h" +#include "flutter/fml/trace_event.h" +#include "third_party/skia/include/codec/SkCodec.h" + +namespace flutter { + +ImageDecoder::ImageDecoder( + TaskRunners runners, + std::shared_ptr concurrent_task_runner, + fml::WeakPtr io_manager) + : runners_(std::move(runners)), + concurrent_task_runner_(std::move(concurrent_task_runner)), + io_manager_(std::move(io_manager)), + weak_factory_(this) { + FML_DCHECK(runners_.IsValid()); + FML_DCHECK(runners_.GetUITaskRunner()->RunsTasksOnCurrentThread()) + << "The image decoder must be created & collected on the UI thread."; +} + +ImageDecoder::~ImageDecoder() = default; + +// Get the updated dimensions of the image. If both dimensions are specified, +// use them. If one of them is specified, respect the one that is and use the +// aspect ratio to calculate the other. If neither dimension is specified, use +// intrinsic dimensions of the image. +static SkISize GetResizedDimensions(SkISize current_size, + std::optional target_width, + std::optional target_height) { + if (current_size.isEmpty()) { + return SkISize::MakeEmpty(); + } + + if (target_width && target_height) { + return SkISize::Make(target_width.value(), target_height.value()); + } + + const auto aspect_ratio = + static_cast(current_size.width()) / current_size.height(); + + if (target_width) { + return SkISize::Make(target_width.value(), + target_width.value() / aspect_ratio); + } + + if (target_height) { + return SkISize::Make(target_height.value() * aspect_ratio, + target_height.value()); + } + + return current_size; +} + +static sk_sp ResizeRasterImage(sk_sp image, + std::optional target_width, + std::optional target_height, + const fml::tracing::TraceFlow& flow) { + FML_DCHECK(!image->isTextureBacked()); + + const auto resized_dimensions = + GetResizedDimensions(image->dimensions(), target_width, target_height); + + if (resized_dimensions.isEmpty()) { + FML_LOG(ERROR) << "Could not resize to empty dimensions."; + return nullptr; + } + + if (resized_dimensions == image->dimensions()) { + // The resized dimesions are the same as the intrinsic dimensions of the + // image. There is nothing to do. + return image; + } + + TRACE_EVENT0("flutter", __FUNCTION__); + flow.Step(__FUNCTION__); + + const auto scaled_image_info = image->imageInfo().makeWH( + resized_dimensions.width(), resized_dimensions.height()); + + SkBitmap scaled_bitmap; + if (!scaled_bitmap.tryAllocPixels(scaled_image_info)) { + FML_LOG(ERROR) << "Could not allocate bitmap when attempting to scale."; + return nullptr; + } + + if (!image->scalePixels(scaled_bitmap.pixmap(), kLow_SkFilterQuality, + SkImage::kDisallow_CachingHint)) { + FML_LOG(ERROR) << "Could not scale pixels"; + return nullptr; + } + + // Marking this as immutable makes the MakeFromBitmap call share the pixels + // instead of copying. + scaled_bitmap.setImmutable(); + + auto scaled_image = SkImage::MakeFromBitmap(scaled_bitmap); + if (!scaled_image) { + FML_LOG(ERROR) << "Could not create a scaled image from a scaled bitmap."; + return nullptr; + } + + return scaled_image; +} + +static sk_sp ImageFromDecompressedData( + sk_sp data, + ImageDecoder::ImageInfo info, + std::optional target_width, + std::optional target_height, + const fml::tracing::TraceFlow& flow) { + TRACE_EVENT0("flutter", __FUNCTION__); + flow.Step(__FUNCTION__); + auto image = SkImage::MakeRasterData(info.sk_info, data, info.row_bytes); + + if (!image) { + FML_LOG(ERROR) << "Could not create image from decompressed bytes."; + return nullptr; + } + + return ResizeRasterImage(std::move(image), target_width, target_height, flow); +} + +static sk_sp ImageFromCompressedData( + sk_sp data, + std::optional target_width, + std::optional target_height, + const fml::tracing::TraceFlow& flow) { + TRACE_EVENT0("flutter", __FUNCTION__); + flow.Step(__FUNCTION__); + + auto codec = SkCodec::MakeFromData(data); + + if (!codec) { + return nullptr; + } + + const auto encoded_info = codec->getInfo(); + + if (encoded_info.dimensions().isEmpty()) { + return nullptr; + } + + const double desired_width = + target_width.value_or(encoded_info.dimensions().width()); + const double desired_height = + target_height.value_or(encoded_info.dimensions().height()); + + const auto scale_x = desired_width / encoded_info.dimensions().width(); + const auto scale_y = desired_height / encoded_info.dimensions().height(); + + const double scale = std::min({scale_x, scale_y, 1.0}); + + if (scale <= 0.0) { + return nullptr; + } + + // We ask the codec for one of the natively supported dimensions. This may not + // be exactly what we need, but it will also be smaller than 1:1. We will + // still have to perform a resize, but from a smaller intermediate buffer. In + // case no resize needs to be performed, ResizeRasterImage will just return + // the original image. + const auto scaled_dimensions = codec->getScaledDimensions(scale); + + if (scaled_dimensions.isEmpty()) { + return nullptr; + } + + const auto decoded_info = encoded_info.makeWH(scaled_dimensions.width(), + scaled_dimensions.height()); + + SkBitmap decoded_bitmap; + if (!decoded_bitmap.tryAllocPixels(decoded_info)) { + FML_LOG(ERROR) << "Could not perform allocation for image decoding."; + return nullptr; + } + + const auto decompression_result = codec->getPixels(decoded_bitmap.pixmap()); + if (decompression_result != SkCodec::Result::kSuccess) { + FML_LOG(ERROR) << "Could not perform image decompression. Error: " + << SkCodec::ResultToString(decompression_result); + return nullptr; + } + + decoded_bitmap.setImmutable(); + + auto decoded_image = SkImage::MakeFromBitmap(decoded_bitmap); + + if (!decoded_image) { + return nullptr; + } + + return ResizeRasterImage(decoded_image, target_width, target_height, flow); +} + +static SkiaGPUObject UploadRasterImage( + sk_sp image, + fml::WeakPtr context, + fml::RefPtr queue, + const fml::tracing::TraceFlow& flow) { + TRACE_EVENT0("flutter", __FUNCTION__); + flow.Step(__FUNCTION__); + + // Should not already be a texture image because that is the entire point of + // the this method. + FML_DCHECK(!image->isTextureBacked()); + + if (!context || !queue) { + FML_LOG(ERROR) + << "Could not acquire context of release queue for texture upload."; + return {}; + } + + SkPixmap pixmap; + if (!image->peekPixels(&pixmap)) { + FML_LOG(ERROR) << "Could not peek pixels of image for texture upload."; + return {}; + } + + auto texture_image = + SkImage::MakeCrossContextFromPixmap(context.get(), // context + pixmap, // pixmap + true, // buildMips, + nullptr, // dstColorSpace, + true // limitToMaxTextureSize + ); + + if (!texture_image) { + FML_LOG(ERROR) << "Could not make x-context image."; + return {}; + } + + return {texture_image, queue}; +} + +void ImageDecoder::Decode(ImageDescriptor descriptor, ImageResult callback) { + TRACE_EVENT0("flutter", __FUNCTION__); + fml::tracing::TraceFlow flow(__FUNCTION__); + + FML_DCHECK(callback); + FML_DCHECK(runners_.GetUITaskRunner()->RunsTasksOnCurrentThread()); + + // Always service the callback on the UI thread. + auto result = [callback, ui_runner = runners_.GetUITaskRunner()]( + SkiaGPUObject image, + fml::tracing::TraceFlow flow) { + ui_runner->PostTask(fml::MakeCopyable( + [callback, image = std::move(image), flow = std::move(flow)]() mutable { + // We are going to terminate the trace flow here. Flows cannot + // terminate without a base trace. Add one explicitly. + TRACE_EVENT0("flutter", "ImageDecodeCallback"); + flow.End(); + callback(std::move(image)); + })); + }; + + if (!descriptor.data || descriptor.data->size() == 0) { + result({}, std::move(flow)); + return; + } + + concurrent_task_runner_->PostTask( + fml::MakeCopyable([descriptor, // + io_manager = io_manager_, // + io_runner = runners_.GetIOTaskRunner(), // + result, // + flow = std::move(flow) // + ]() mutable { + // Step 1: Decompress the image. + // On Worker. + + auto decompressed = + descriptor.decompressed_image_info + ? ImageFromDecompressedData( + std::move(descriptor.data), // + descriptor.decompressed_image_info.value(), // + descriptor.target_width, // + descriptor.target_height, // + flow // + ) + : ImageFromCompressedData(std::move(descriptor.data), // + descriptor.target_width, // + descriptor.target_height, // + flow); + + if (!decompressed) { + FML_LOG(ERROR) << "Could not decompress image."; + result({}, std::move(flow)); + return; + } + + // Step 2: Update the image to the GPU. + // On IO Thread. + + io_runner->PostTask(fml::MakeCopyable([io_manager, decompressed, result, + flow = + std::move(flow)]() mutable { + if (!io_manager) { + FML_LOG(ERROR) << "Could not acquire IO manager."; + return result({}, std::move(flow)); + } + + // If the IO manager does not have a resource context, the caller + // might not have set one or a software backend could be in use. + // Either way, just return the image as-is. + if (!io_manager->GetResourceContext()) { + result({std::move(decompressed), io_manager->GetSkiaUnrefQueue()}, + std::move(flow)); + return; + } + + auto uploaded = UploadRasterImage( + std::move(decompressed), io_manager->GetResourceContext(), + io_manager->GetSkiaUnrefQueue(), flow); + + if (!uploaded.get()) { + FML_LOG(ERROR) << "Could not upload image to the GPU."; + result({}, std::move(flow)); + return; + } + + // Finally, all done. + result(std::move(uploaded), std::move(flow)); + })); + })); +} + +fml::WeakPtr ImageDecoder::GetWeakPtr() const { + return weak_factory_.GetWeakPtr(); +} + +} // namespace flutter diff --git a/lib/ui/painting/image_decoder.h b/lib/ui/painting/image_decoder.h new file mode 100644 index 0000000000000..202dff569e1a5 --- /dev/null +++ b/lib/ui/painting/image_decoder.h @@ -0,0 +1,73 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_UI_PAINTING_IMAGE_DECODER_H_ +#define FLUTTER_LIB_UI_PAINTING_IMAGE_DECODER_H_ + +#include +#include + +#include "flutter/common/task_runners.h" +#include "flutter/flow/skia_gpu_object.h" +#include "flutter/fml/concurrent_message_loop.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/lib/ui/io_manager.h" +#include "third_party/skia/include/core/SkData.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkImageInfo.h" +#include "third_party/skia/include/core/SkRefCnt.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace flutter { + +// An object that coordinates image decompression and texture upload across +// multiple threads/components in the shell. This object must be created, +// accessed and collected on the UI thread (typically the engine or its runtime +// controller). None of the expensive operations performed by this component +// occur in a frame pipeline. +class ImageDecoder { + public: + ImageDecoder( + TaskRunners runners, + std::shared_ptr concurrent_task_runner, + fml::WeakPtr io_manager); + + ~ImageDecoder(); + + struct ImageInfo { + SkImageInfo sk_info = {}; + size_t row_bytes = 0; + }; + + struct ImageDescriptor { + sk_sp data; + std::optional decompressed_image_info; + std::optional target_width; + std::optional target_height; + }; + + using ImageResult = std::function)>; + + // Takes an image descriptor and returns a handle to a texture resident on the + // GPU. All image decompression and resizes are done on a worker thread + // concurrently. Texture upload is done on the IO thread and the result + // returned back on the UI thread. On error, the texture is null but the + // callback is guaranteed to return on the UI thread. + void Decode(ImageDescriptor descriptor, ImageResult result); + + fml::WeakPtr GetWeakPtr() const; + + private: + TaskRunners runners_; + std::shared_ptr concurrent_task_runner_; + fml::WeakPtr io_manager_; + fml::WeakPtrFactory weak_factory_; + + FML_DISALLOW_COPY_AND_ASSIGN(ImageDecoder); +}; + +} // namespace flutter + +#endif // FLUTTER_LIB_UI_PAINTING_IMAGE_DECODER_H_ diff --git a/lib/ui/painting/image_decoder_unittests.cc b/lib/ui/painting/image_decoder_unittests.cc new file mode 100644 index 0000000000000..7fb08ff50e496 --- /dev/null +++ b/lib/ui/painting/image_decoder_unittests.cc @@ -0,0 +1,390 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/common/task_runners.h" +#include "flutter/fml/mapping.h" +#include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/lib/ui/painting/image_decoder.h" +#include "flutter/testing/test_gl_surface.h" +#include "flutter/testing/testing.h" +#include "flutter/testing/thread_test.h" + +namespace flutter { +namespace testing { + +using ImageDecoderFixtureTest = ThreadTest; + +class TestIOManager final : public IOManager { + public: + TestIOManager(fml::RefPtr task_runner, + bool has_gpu_context = true) + : gl_context_(has_gpu_context ? gl_surface_.CreateContext() : nullptr), + weak_gl_context_factory_( + has_gpu_context ? std::make_unique>( + gl_context_.get()) + : nullptr), + unref_queue_(fml::MakeRefCounted( + task_runner, + fml::TimeDelta::FromNanoseconds(0))), + runner_(task_runner), + weak_factory_(this) { + FML_CHECK(task_runner->RunsTasksOnCurrentThread()) + << "The IO manager must be initialized its primary task runner. The " + "test harness may not be setup correctly/safely."; + weak_prototype_ = weak_factory_.GetWeakPtr(); + } + + ~TestIOManager() override { + fml::AutoResetWaitableEvent latch; + fml::TaskRunner::RunNowOrPostTask(runner_, + [&latch, queue = unref_queue_]() { + queue->Drain(); + latch.Signal(); + }); + latch.Wait(); + } + + // |IOManager| + fml::WeakPtr GetWeakIOManager() const override { + return weak_prototype_; + } + + // |IOManager| + fml::WeakPtr GetResourceContext() const override { + return weak_gl_context_factory_ ? weak_gl_context_factory_->GetWeakPtr() + : fml::WeakPtr{}; + } + + // |IOManager| + fml::RefPtr GetSkiaUnrefQueue() const override { + return unref_queue_; + } + + private: + TestGLSurface gl_surface_; + sk_sp gl_context_; + std::unique_ptr> weak_gl_context_factory_; + fml::RefPtr unref_queue_; + fml::WeakPtr weak_prototype_; + fml::RefPtr runner_; + fml::WeakPtrFactory weak_factory_; + + FML_DISALLOW_COPY_AND_ASSIGN(TestIOManager); +}; + +static sk_sp OpenFixtureAsSkData(const char* name) { + auto fixtures_directory = + fml::OpenFile(GetFixturesPath(), false, fml::FilePermission::kRead); + if (!fixtures_directory.is_valid()) { + return nullptr; + } + + auto fixture_mapping = + fml::FileMapping::CreateReadOnly(fixtures_directory, name); + + if (!fixture_mapping) { + return nullptr; + } + + SkData::ReleaseProc on_release = [](const void* ptr, void* context) -> void { + delete reinterpret_cast(context); + }; + + return SkData::MakeWithProc(fixture_mapping->GetMapping(), + fixture_mapping->GetSize(), on_release, + fixture_mapping.release()); +} + +TEST_F(ImageDecoderFixtureTest, CanCreateImageDecoder) { + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + GetThreadTaskRunner(), // gpu + GetThreadTaskRunner(), // ui + GetThreadTaskRunner() // io + + ); + + fml::AutoResetWaitableEvent latch; + runners.GetIOTaskRunner()->PostTask([&]() { + TestIOManager manager(runners.GetIOTaskRunner()); + ImageDecoder decoder(std::move(runners), loop->GetTaskRunner(), + manager.GetWeakIOManager()); + latch.Signal(); + }); + latch.Wait(); +} + +TEST_F(ImageDecoderFixtureTest, InvalidImageResultsError) { + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + GetThreadTaskRunner(), // gpu + GetThreadTaskRunner(), // ui + GetThreadTaskRunner() // io + ); + + fml::AutoResetWaitableEvent latch; + GetThreadTaskRunner()->PostTask([&]() { + TestIOManager manager(runners.GetIOTaskRunner()); + ImageDecoder decoder(runners, loop->GetTaskRunner(), + manager.GetWeakIOManager()); + + ImageDecoder::ImageDescriptor image_descriptor; + image_descriptor.data = OpenFixtureAsSkData("ThisDoesNotExist.jpg"); + + ASSERT_FALSE(image_descriptor.data); + + ImageDecoder::ImageResult callback = [&](SkiaGPUObject image) { + ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread()); + ASSERT_FALSE(image.get()); + latch.Signal(); + }; + decoder.Decode(std::move(image_descriptor), callback); + }); + latch.Wait(); +} + +TEST_F(ImageDecoderFixtureTest, ValidImageResultsInSuccess) { + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + CreateNewThread("gpu"), // gpu + CreateNewThread("ui"), // ui + CreateNewThread("io") // io + ); + + fml::AutoResetWaitableEvent latch; + + std::unique_ptr io_manager; + std::unique_ptr image_decoder; + + auto decode_image = [&]() { + image_decoder = std::make_unique( + runners, loop->GetTaskRunner(), io_manager->GetWeakIOManager()); + + ImageDecoder::ImageDescriptor image_descriptor; + image_descriptor.data = OpenFixtureAsSkData("DashInNooglerHat.jpg"); + + ASSERT_TRUE(image_descriptor.data); + ASSERT_GE(image_descriptor.data->size(), 0u); + + ImageDecoder::ImageResult callback = [&](SkiaGPUObject image) { + ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread()); + ASSERT_TRUE(image.get()); + latch.Signal(); + }; + image_decoder->Decode(std::move(image_descriptor), callback); + }; + + auto setup_io_manager_and_decode = [&]() { + io_manager = std::make_unique(runners.GetIOTaskRunner()); + runners.GetUITaskRunner()->PostTask(decode_image); + }; + + runners.GetIOTaskRunner()->PostTask(setup_io_manager_and_decode); + + latch.Wait(); +} + +TEST_F(ImageDecoderFixtureTest, CanDecodeWithoutAGPUContext) { + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + CreateNewThread("gpu"), // gpu + CreateNewThread("ui"), // ui + CreateNewThread("io") // io + ); + + fml::AutoResetWaitableEvent latch; + + std::unique_ptr io_manager; + std::unique_ptr image_decoder; + + auto decode_image = [&]() { + image_decoder = std::make_unique( + runners, loop->GetTaskRunner(), io_manager->GetWeakIOManager()); + + ImageDecoder::ImageDescriptor image_descriptor; + image_descriptor.data = OpenFixtureAsSkData("DashInNooglerHat.jpg"); + + ASSERT_TRUE(image_descriptor.data); + ASSERT_GE(image_descriptor.data->size(), 0u); + + ImageDecoder::ImageResult callback = [&](SkiaGPUObject image) { + ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread()); + ASSERT_TRUE(image.get()); + latch.Signal(); + }; + image_decoder->Decode(std::move(image_descriptor), callback); + }; + + auto setup_io_manager_and_decode = [&]() { + io_manager = + std::make_unique(runners.GetIOTaskRunner(), false); + runners.GetUITaskRunner()->PostTask(decode_image); + }; + + runners.GetIOTaskRunner()->PostTask(setup_io_manager_and_decode); + + latch.Wait(); +} + +TEST_F(ImageDecoderFixtureTest, CanDecodeWithResizes) { + const auto image_dimensions = + SkImage::MakeFromEncoded(OpenFixtureAsSkData("DashInNooglerHat.jpg")) + ->dimensions(); + + ASSERT_FALSE(image_dimensions.isEmpty()); + + ASSERT_NE(image_dimensions.width(), image_dimensions.height()); + + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + CreateNewThread("gpu"), // gpu + CreateNewThread("ui"), // ui + CreateNewThread("io") // io + ); + + fml::AutoResetWaitableEvent latch; + std::unique_ptr io_manager; + std::unique_ptr image_decoder; + + // Setup the IO manager. + runners.GetIOTaskRunner()->PostTask([&]() { + io_manager = std::make_unique(runners.GetIOTaskRunner()); + latch.Signal(); + }); + latch.Wait(); + + // Setup the image decoder. + runners.GetUITaskRunner()->PostTask([&]() { + image_decoder = std::make_unique( + runners, loop->GetTaskRunner(), io_manager->GetWeakIOManager()); + + latch.Signal(); + }); + latch.Wait(); + + // Setup a generic decoding utility that gives us the final decoded size. + auto decoded_size = [&](std::optional target_width, + std::optional target_height) -> SkISize { + SkISize final_size = SkISize::MakeEmpty(); + runners.GetUITaskRunner()->PostTask([&]() { + ImageDecoder::ImageDescriptor image_descriptor; + image_descriptor.target_width = target_width; + image_descriptor.target_height = target_height; + image_descriptor.data = OpenFixtureAsSkData("DashInNooglerHat.jpg"); + + ASSERT_TRUE(image_descriptor.data); + ASSERT_GE(image_descriptor.data->size(), 0u); + + ImageDecoder::ImageResult callback = [&](SkiaGPUObject image) { + ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread()); + ASSERT_TRUE(image.get()); + final_size = image.get()->dimensions(); + latch.Signal(); + }; + image_decoder->Decode(std::move(image_descriptor), callback); + }); + latch.Wait(); + return final_size; + }; + + ASSERT_EQ(SkISize::Make(3024, 4032), image_dimensions); + ASSERT_EQ(decoded_size({}, {}), image_dimensions); + ASSERT_EQ(decoded_size(100, {}), SkISize::Make(100, 133)); + ASSERT_EQ(decoded_size({}, 100), SkISize::Make(75, 100)); + ASSERT_EQ(decoded_size(100, 100), SkISize::Make(100, 100)); +} + +TEST_F(ImageDecoderFixtureTest, CanResizeWithoutDecode) { + ImageDecoder::ImageInfo info = {}; + sk_sp decompressed_data; + SkISize image_dimensions = SkISize::MakeEmpty(); + { + auto image = + SkImage::MakeFromEncoded(OpenFixtureAsSkData("DashInNooglerHat.jpg")) + ->makeRasterImage(); + image_dimensions = image->dimensions(); + SkPixmap pixmap; + ASSERT_TRUE(image->peekPixels(&pixmap)); + info.sk_info = SkImageInfo::MakeN32Premul(image_dimensions); + info.row_bytes = pixmap.rowBytes(); + decompressed_data = + SkData::MakeWithCopy(pixmap.writable_addr(), pixmap.computeByteSize()); + } + + // This is not susecptible to changes in the underlying image decoder. + ASSERT_EQ(decompressed_data->size(), 48771072u); + ASSERT_EQ(decompressed_data->size(), + image_dimensions.width() * image_dimensions.height() * 4u); + ASSERT_EQ(info.row_bytes, image_dimensions.width() * 4u); + ASSERT_FALSE(image_dimensions.isEmpty()); + ASSERT_NE(image_dimensions.width(), image_dimensions.height()); + + auto loop = fml::ConcurrentMessageLoop::Create(); + TaskRunners runners(GetCurrentTestName(), // label + GetThreadTaskRunner(), // platform + CreateNewThread("gpu"), // gpu + CreateNewThread("ui"), // ui + CreateNewThread("io") // io + ); + + fml::AutoResetWaitableEvent latch; + std::unique_ptr io_manager; + std::unique_ptr image_decoder; + + // Setup the IO manager. + runners.GetIOTaskRunner()->PostTask([&]() { + io_manager = std::make_unique(runners.GetIOTaskRunner()); + latch.Signal(); + }); + latch.Wait(); + + // Setup the image decoder. + runners.GetUITaskRunner()->PostTask([&]() { + image_decoder = std::make_unique( + runners, loop->GetTaskRunner(), io_manager->GetWeakIOManager()); + + latch.Signal(); + }); + latch.Wait(); + + // Setup a generic decoding utility that gives us the final decoded size. + auto decoded_size = [&](std::optional target_width, + std::optional target_height) -> SkISize { + SkISize final_size = SkISize::MakeEmpty(); + runners.GetUITaskRunner()->PostTask([&]() { + ImageDecoder::ImageDescriptor image_descriptor; + image_descriptor.target_width = target_width; + image_descriptor.target_height = target_height; + image_descriptor.data = decompressed_data; + image_descriptor.decompressed_image_info = info; + + ASSERT_TRUE(image_descriptor.data); + ASSERT_GE(image_descriptor.data->size(), 0u); + + ImageDecoder::ImageResult callback = [&](SkiaGPUObject image) { + ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread()); + ASSERT_TRUE(image.get()); + final_size = image.get()->dimensions(); + latch.Signal(); + }; + image_decoder->Decode(std::move(image_descriptor), callback); + }); + latch.Wait(); + return final_size; + }; + + ASSERT_EQ(SkISize::Make(3024, 4032), image_dimensions); + ASSERT_EQ(decoded_size({}, {}), image_dimensions); + ASSERT_EQ(decoded_size(100, {}), SkISize::Make(100, 133)); + ASSERT_EQ(decoded_size({}, 100), SkISize::Make(75, 100)); + ASSERT_EQ(decoded_size(100, 100), SkISize::Make(100, 100)); +} + +} // namespace testing +} // namespace flutter diff --git a/lib/ui/painting/multi_frame_codec.cc b/lib/ui/painting/multi_frame_codec.cc new file mode 100644 index 0000000000000..0cd29b6623e02 --- /dev/null +++ b/lib/ui/painting/multi_frame_codec.cc @@ -0,0 +1,191 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/ui/painting/multi_frame_codec.h" + +#include "flutter/fml/make_copyable.h" +#include "third_party/dart/runtime/include/dart_api.h" +#include "third_party/skia/include/core/SkPixelRef.h" +#include "third_party/tonic/logging/dart_invoke.h" + +namespace flutter { + +MultiFrameCodec::MultiFrameCodec(std::unique_ptr codec) + : codec_(std::move(codec)), + frameCount_(codec_->getFrameCount()), + repetitionCount_(codec_->getRepetitionCount()), + nextFrameIndex_(0) {} + +MultiFrameCodec::~MultiFrameCodec() = default; + +static void InvokeNextFrameCallback( + fml::RefPtr frameInfo, + std::unique_ptr callback, + size_t trace_id) { + std::shared_ptr dart_state = callback->dart_state().lock(); + if (!dart_state) { + FML_DLOG(ERROR) << "Could not acquire Dart state while attempting to fire " + "next frame callback."; + return; + } + tonic::DartState::Scope scope(dart_state); + if (!frameInfo) { + tonic::DartInvoke(callback->value(), {Dart_Null()}); + } else { + tonic::DartInvoke(callback->value(), {ToDart(frameInfo)}); + } +} + +// Copied the source bitmap to the destination. If this cannot occur due to +// running out of memory or the image info not being compatible, returns false. +static bool CopyToBitmap(SkBitmap* dst, + SkColorType dstColorType, + const SkBitmap& src) { + SkPixmap srcPM; + if (!src.peekPixels(&srcPM)) { + return false; + } + + SkBitmap tmpDst; + SkImageInfo dstInfo = srcPM.info().makeColorType(dstColorType); + if (!tmpDst.setInfo(dstInfo)) { + return false; + } + + if (!tmpDst.tryAllocPixels()) { + return false; + } + + SkPixmap dstPM; + if (!tmpDst.peekPixels(&dstPM)) { + return false; + } + + if (!srcPM.readPixels(dstPM)) { + return false; + } + + dst->swap(tmpDst); + return true; +} + +sk_sp MultiFrameCodec::GetNextFrameImage( + fml::WeakPtr resourceContext) { + SkBitmap bitmap = SkBitmap(); + SkImageInfo info = codec_->getInfo().makeColorType(kN32_SkColorType); + if (info.alphaType() == kUnpremul_SkAlphaType) { + info = info.makeAlphaType(kPremul_SkAlphaType); + } + bitmap.allocPixels(info); + + SkCodec::Options options; + options.fFrameIndex = nextFrameIndex_; + SkCodec::FrameInfo frameInfo; + codec_->getFrameInfo(nextFrameIndex_, &frameInfo); + const int requiredFrameIndex = frameInfo.fRequiredFrame; + if (requiredFrameIndex != SkCodec::kNoFrame) { + if (lastRequiredFrame_ == nullptr) { + FML_LOG(ERROR) << "Frame " << nextFrameIndex_ << " depends on frame " + << requiredFrameIndex + << " and no required frames are cached."; + return nullptr; + } else if (lastRequiredFrameIndex_ != requiredFrameIndex) { + FML_DLOG(INFO) << "Required frame " << requiredFrameIndex + << " is not cached. Using " << lastRequiredFrameIndex_ + << " instead"; + } + + if (lastRequiredFrame_->getPixels() && + CopyToBitmap(&bitmap, lastRequiredFrame_->colorType(), + *lastRequiredFrame_)) { + options.fPriorFrame = requiredFrameIndex; + } + } + + if (SkCodec::kSuccess != codec_->getPixels(info, bitmap.getPixels(), + bitmap.rowBytes(), &options)) { + FML_LOG(ERROR) << "Could not getPixels for frame " << nextFrameIndex_; + return nullptr; + } + + // Hold onto this if we need it to decode future frames. + if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kKeep) { + lastRequiredFrame_ = std::make_unique(bitmap); + lastRequiredFrameIndex_ = nextFrameIndex_; + } + + if (resourceContext) { + SkPixmap pixmap(bitmap.info(), bitmap.pixelRef()->pixels(), + bitmap.pixelRef()->rowBytes()); + // This indicates that we do not want a "linear blending" decode. + sk_sp dstColorSpace = nullptr; + return SkImage::MakeCrossContextFromPixmap(resourceContext.get(), pixmap, + true, dstColorSpace.get()); + } else { + // Defer decoding until time of draw later on the GPU thread. Can happen + // when GL operations are currently forbidden such as in the background + // on iOS. + return SkImage::MakeFromBitmap(bitmap); + } +} + +void MultiFrameCodec::GetNextFrameAndInvokeCallback( + std::unique_ptr callback, + fml::RefPtr ui_task_runner, + fml::WeakPtr resourceContext, + fml::RefPtr unref_queue, + size_t trace_id) { + fml::RefPtr frameInfo = NULL; + sk_sp skImage = GetNextFrameImage(resourceContext); + if (skImage) { + fml::RefPtr image = CanvasImage::Create(); + image->set_image({skImage, std::move(unref_queue)}); + SkCodec::FrameInfo skFrameInfo; + codec_->getFrameInfo(nextFrameIndex_, &skFrameInfo); + frameInfo = + fml::MakeRefCounted(std::move(image), skFrameInfo.fDuration); + } + nextFrameIndex_ = (nextFrameIndex_ + 1) % frameCount_; + + ui_task_runner->PostTask(fml::MakeCopyable( + [callback = std::move(callback), frameInfo, trace_id]() mutable { + InvokeNextFrameCallback(frameInfo, std::move(callback), trace_id); + })); +} + +Dart_Handle MultiFrameCodec::getNextFrame(Dart_Handle callback_handle) { + static size_t trace_counter = 1; + const size_t trace_id = trace_counter++; + + if (!Dart_IsClosure(callback_handle)) { + return tonic::ToDart("Callback must be a function"); + } + + auto* dart_state = UIDartState::Current(); + + const auto& task_runners = dart_state->GetTaskRunners(); + + task_runners.GetIOTaskRunner()->PostTask(fml::MakeCopyable( + [callback = std::make_unique( + tonic::DartState::Current(), callback_handle), + this, trace_id, ui_task_runner = task_runners.GetUITaskRunner(), + queue = UIDartState::Current()->GetSkiaUnrefQueue(), + context = dart_state->GetResourceContext()]() mutable { + GetNextFrameAndInvokeCallback(std::move(callback), + std::move(ui_task_runner), context, + std::move(queue), trace_id); + })); + + return Dart_Null(); +} + +int MultiFrameCodec::frameCount() const { + return frameCount_; +} + +int MultiFrameCodec::repetitionCount() const { + return repetitionCount_; +} + +} // namespace flutter diff --git a/lib/ui/painting/multi_frame_codec.h b/lib/ui/painting/multi_frame_codec.h new file mode 100644 index 0000000000000..0be63c2a57582 --- /dev/null +++ b/lib/ui/painting/multi_frame_codec.h @@ -0,0 +1,54 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_UI_PAINTING_MUTLI_FRAME_CODEC_H_ +#define FLUTTER_LIB_UI_PAINTING_MUTLI_FRAME_CODEC_H_ + +#include "flutter/fml/macros.h" +#include "flutter/lib/ui/painting/codec.h" + +namespace flutter { + +class MultiFrameCodec : public Codec { + public: + MultiFrameCodec(std::unique_ptr codec); + + ~MultiFrameCodec() override; + + // |Codec| + int frameCount() const override; + + // |Codec| + int repetitionCount() const override; + + // |Codec| + Dart_Handle getNextFrame(Dart_Handle args) override; + + private: + const std::unique_ptr codec_; + const int frameCount_; + const int repetitionCount_; + int nextFrameIndex_; + + // The last decoded frame that's required to decode any subsequent frames. + std::unique_ptr lastRequiredFrame_; + // The index of the last decoded required frame. + int lastRequiredFrameIndex_ = -1; + + sk_sp GetNextFrameImage(fml::WeakPtr resourceContext); + + void GetNextFrameAndInvokeCallback( + std::unique_ptr callback, + fml::RefPtr ui_task_runner, + fml::WeakPtr resourceContext, + fml::RefPtr unref_queue, + size_t trace_id); + + FML_FRIEND_MAKE_REF_COUNTED(MultiFrameCodec); + FML_FRIEND_REF_COUNTED_THREAD_SAFE(MultiFrameCodec); +}; + +} // namespace flutter + +#endif // FLUTTER_LIB_UI_PAINTING_MUTLI_FRAME_CODEC_H_ diff --git a/lib/ui/painting/single_frame_codec.cc b/lib/ui/painting/single_frame_codec.cc new file mode 100644 index 0000000000000..8036f767ead7b --- /dev/null +++ b/lib/ui/painting/single_frame_codec.cc @@ -0,0 +1,79 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/ui/painting/single_frame_codec.h" + +#include "flutter/lib/ui/painting/frame_info.h" +#include "flutter/lib/ui/ui_dart_state.h" +#include "third_party/tonic/logging/dart_invoke.h" + +namespace flutter { + +SingleFrameCodec::SingleFrameCodec(ImageDecoder::ImageDescriptor descriptor) + : descriptor_(std::move(descriptor)) {} + +SingleFrameCodec::~SingleFrameCodec() = default; + +int SingleFrameCodec::frameCount() const { + return 1; +} + +int SingleFrameCodec::repetitionCount() const { + return 0; +} + +Dart_Handle SingleFrameCodec::getNextFrame(Dart_Handle callback_handle) { + if (!Dart_IsClosure(callback_handle)) { + return tonic::ToDart("Callback must be a function"); + } + + // This has to be valid because this method is called from Dart. + auto dart_state = UIDartState::Current(); + + auto decoder = dart_state->GetImageDecoder(); + + if (!decoder) { + return tonic::ToDart("Image decoder not available."); + } + + auto raw_callback = new DartPersistentValue(dart_state, callback_handle); + + // We dont want to to put the raw callback in a lambda capture because we have + // to mutate (i.e destroy) it in the callback. Using MakeCopyable will create + // a shared pointer for the captures which can be destroyed on any thread. But + // we have to ensure that the DartPersistentValue is only destroyed on the UI + // thread. + decoder->Decode(descriptor_, [raw_callback](auto image) { + std::unique_ptr callback(raw_callback); + + auto state = callback->dart_state().lock(); + + if (!state) { + // This is probably because the isolate has been terminated before the + // image could be decoded. + + return; + } + + tonic::DartState::Scope scope(state.get()); + + auto canvas_image = fml::MakeRefCounted(); + canvas_image->set_image(std::move(image)); + + auto frame_info = fml::MakeRefCounted(std::move(canvas_image), + 0 /* duration */); + + tonic::DartInvoke(callback->value(), {tonic::ToDart(frame_info)}); + }); + + return Dart_Null(); +} + +size_t SingleFrameCodec::GetAllocationSize() { + const auto& data = descriptor_.data; + auto data_byte_size = data ? data->size() : 0; + return data_byte_size + sizeof(this); +} + +} // namespace flutter diff --git a/lib/ui/painting/single_frame_codec.h b/lib/ui/painting/single_frame_codec.h new file mode 100644 index 0000000000000..9684eef11219b --- /dev/null +++ b/lib/ui/painting/single_frame_codec.h @@ -0,0 +1,41 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_UI_PAINTING_SINGLE_FRAME_CODEC_H_ +#define FLUTTER_LIB_UI_PAINTING_SINGLE_FRAME_CODEC_H_ + +#include "flutter/fml/macros.h" +#include "flutter/lib/ui/painting/codec.h" +#include "flutter/lib/ui/painting/image_decoder.h" + +namespace flutter { + +class SingleFrameCodec : public Codec { + public: + SingleFrameCodec(ImageDecoder::ImageDescriptor descriptor); + + ~SingleFrameCodec() override; + + // |Codec| + int frameCount() const override; + + // |Codec| + int repetitionCount() const override; + + // |Codec| + Dart_Handle getNextFrame(Dart_Handle args) override; + + // |DartWrappable| + size_t GetAllocationSize() override; + + private: + ImageDecoder::ImageDescriptor descriptor_; + + FML_FRIEND_MAKE_REF_COUNTED(SingleFrameCodec); + FML_FRIEND_REF_COUNTED_THREAD_SAFE(SingleFrameCodec); +}; + +} // namespace flutter + +#endif // FLUTTER_LIB_UI_PAINTING_SINGLE_FRAME_CODEC_H_ diff --git a/lib/ui/ui_dart_state.cc b/lib/ui/ui_dart_state.cc index b26e01afb1625..8b84d18921957 100644 --- a/lib/ui/ui_dart_state.cc +++ b/lib/ui/ui_dart_state.cc @@ -19,6 +19,7 @@ UIDartState::UIDartState( TaskObserverRemove remove_callback, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, std::string logger_prefix, @@ -29,6 +30,7 @@ UIDartState::UIDartState( remove_callback_(std::move(remove_callback)), snapshot_delegate_(std::move(snapshot_delegate)), io_manager_(std::move(io_manager)), + image_decoder_(std::move(image_decoder)), advisory_script_uri_(std::move(advisory_script_uri)), advisory_script_entrypoint_(std::move(advisory_script_entrypoint)), logger_prefix_(std::move(logger_prefix)), @@ -124,6 +126,10 @@ fml::WeakPtr UIDartState::GetResourceContext() const { return io_manager_->GetResourceContext(); } +fml::WeakPtr UIDartState::GetImageDecoder() const { + return image_decoder_; +} + std::shared_ptr UIDartState::GetIsolateNameServer() const { return isolate_name_server_; } diff --git a/lib/ui/ui_dart_state.h b/lib/ui/ui_dart_state.h index 4f9c33387ec13..550c72ff98178 100644 --- a/lib/ui/ui_dart_state.h +++ b/lib/ui/ui_dart_state.h @@ -16,6 +16,7 @@ #include "flutter/fml/memory/weak_ptr.h" #include "flutter/lib/ui/io_manager.h" #include "flutter/lib/ui/isolate_name_server/isolate_name_server.h" +#include "flutter/lib/ui/painting/image_decoder.h" #include "flutter/lib/ui/snapshot_delegate.h" #include "third_party/dart/runtime/include/dart_api.h" #include "third_party/skia/include/gpu/GrContext.h" @@ -53,6 +54,8 @@ class UIDartState : public tonic::DartState { fml::WeakPtr GetResourceContext() const; + fml::WeakPtr GetImageDecoder() const; + std::shared_ptr GetIsolateNameServer() const; tonic::DartErrorHandleType GetLastError(); @@ -77,6 +80,7 @@ class UIDartState : public tonic::DartState { TaskObserverRemove remove_callback, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, std::string logger_prefix, @@ -99,6 +103,7 @@ class UIDartState : public tonic::DartState { const TaskObserverRemove remove_callback_; fml::WeakPtr snapshot_delegate_; fml::WeakPtr io_manager_; + fml::WeakPtr image_decoder_; const std::string advisory_script_uri_; const std::string advisory_script_entrypoint_; const std::string logger_prefix_; diff --git a/runtime/dart_isolate.cc b/runtime/dart_isolate.cc index 96ac6facd333a..91f39b3aebda1 100644 --- a/runtime/dart_isolate.cc +++ b/runtime/dart_isolate.cc @@ -37,6 +37,7 @@ std::weak_ptr DartIsolate::CreateRootIsolate( std::unique_ptr window, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, Dart_IsolateFlags* flags, @@ -62,6 +63,7 @@ std::weak_ptr DartIsolate::CreateRootIsolate( task_runners, // task runners std::move(snapshot_delegate), // snapshot delegate std::move(io_manager), // IO manager + std::move(image_decoder), // Image Decoder advisory_script_uri, // advisory URI advisory_script_entrypoint, // advisory entrypoint nullptr, // child isolate preparer @@ -106,6 +108,7 @@ DartIsolate::DartIsolate(const Settings& settings, TaskRunners task_runners, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, ChildIsolatePreparer child_isolate_preparer, @@ -116,6 +119,7 @@ DartIsolate::DartIsolate(const Settings& settings, settings.task_observer_remove, std::move(snapshot_delegate), std::move(io_manager), + std::move(image_decoder), advisory_script_uri, advisory_script_entrypoint, settings.log_tag, @@ -597,6 +601,7 @@ Dart_Isolate DartIsolate::DartCreateAndStartServiceIsolate( nullptr, // window {}, // snapshot delegate {}, // IO Manager + {}, // Image Decoder DART_VM_SERVICE_ISOLATE_NAME, // script uri DART_VM_SERVICE_ISOLATE_NAME, // script entrypoint flags, // flags @@ -709,6 +714,7 @@ DartIsolate::CreateDartVMAndEmbedderObjectPair( null_task_runners, // task_runners fml::WeakPtr{}, // snapshot_delegate fml::WeakPtr{}, // io_manager + fml::WeakPtr{}, // io_manager advisory_script_uri, // advisory_script_uri advisory_script_entrypoint, // advisory_script_entrypoint (*raw_embedder_isolate)->child_isolate_preparer_, // preparer diff --git a/runtime/dart_isolate.h b/runtime/dart_isolate.h index 60412972abc93..20f53a9a402bf 100644 --- a/runtime/dart_isolate.h +++ b/runtime/dart_isolate.h @@ -49,6 +49,7 @@ class DartIsolate : public UIDartState { std::unique_ptr window, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, Dart_IsolateFlags* flags, @@ -61,6 +62,7 @@ class DartIsolate : public UIDartState { TaskRunners task_runners, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, ChildIsolatePreparer child_isolate_preparer, diff --git a/runtime/dart_isolate_unittests.cc b/runtime/dart_isolate_unittests.cc index e752582cc99aa..ae14042ecc743 100644 --- a/runtime/dart_isolate_unittests.cc +++ b/runtime/dart_isolate_unittests.cc @@ -43,6 +43,7 @@ TEST_F(DartIsolateTest, RootIsolateCreationAndShutdown) { nullptr, // window {}, // snapshot delegate {}, // io manager + {}, // image decoder "main.dart", // advisory uri "main", // advisory entrypoint, nullptr, // flags @@ -76,6 +77,7 @@ TEST_F(DartIsolateTest, IsolateShutdownCallbackIsInIsolateScope) { nullptr, // window {}, // snapshot delegate {}, // io manager + {}, // image decoder "main.dart", // advisory uri "main", // advisory entrypoint nullptr, // flags @@ -186,6 +188,7 @@ static void RunDartCodeInIsolate(DartVMRef& vm_ref, nullptr, // window {}, // snapshot delegate {}, // io manager + {}, // image decoder "main.dart", // advisory uri "main", // advisory entrypoint nullptr, // flags diff --git a/runtime/dart_lifecycle_unittests.cc b/runtime/dart_lifecycle_unittests.cc index a6de7f792f1cd..eec7aa6c9c4d5 100644 --- a/runtime/dart_lifecycle_unittests.cc +++ b/runtime/dart_lifecycle_unittests.cc @@ -58,6 +58,7 @@ static std::shared_ptr CreateAndRunRootIsolate( {}, // window {}, // snapshot_delegate {}, // io_manager + {}, // image_decoder "main.dart", // advisory_script_uri entrypoint.c_str(), // advisory_script_entrypoint nullptr, // flags diff --git a/runtime/dart_vm.cc b/runtime/dart_vm.cc index 903e74b1591f7..0e4109051e040 100644 --- a/runtime/dart_vm.cc +++ b/runtime/dart_vm.cc @@ -253,6 +253,7 @@ size_t DartVM::GetVMLaunchCount() { DartVM::DartVM(std::shared_ptr vm_data, std::shared_ptr isolate_name_server) : settings_(vm_data->GetSettings()), + concurrent_message_loop_(fml::ConcurrentMessageLoop::Create()), vm_data_(vm_data), isolate_name_server_(std::move(isolate_name_server)), service_protocol_(std::make_shared()) { @@ -451,4 +452,9 @@ std::shared_ptr DartVM::GetIsolateNameServer() const { return isolate_name_server_; } +std::shared_ptr +DartVM::GetConcurrentWorkerTaskRunner() const { + return concurrent_message_loop_->GetTaskRunner(); +} + } // namespace flutter diff --git a/runtime/dart_vm.h b/runtime/dart_vm.h index b88c033b6ef02..9cc2d811ac3a7 100644 --- a/runtime/dart_vm.h +++ b/runtime/dart_vm.h @@ -15,6 +15,7 @@ #include "flutter/fml/memory/ref_counted.h" #include "flutter/fml/memory/ref_ptr.h" #include "flutter/fml/memory/weak_ptr.h" +#include "flutter/fml/message_loop.h" #include "flutter/lib/ui/isolate_name_server/isolate_name_server.h" #include "flutter/runtime/dart_isolate.h" #include "flutter/runtime/dart_snapshot.h" @@ -40,8 +41,12 @@ class DartVM { std::shared_ptr GetIsolateNameServer() const; + std::shared_ptr GetConcurrentWorkerTaskRunner() + const; + private: const Settings settings_; + std::shared_ptr concurrent_message_loop_; std::shared_ptr vm_data_; const std::shared_ptr isolate_name_server_; const std::shared_ptr service_protocol_; diff --git a/runtime/runtime_controller.cc b/runtime/runtime_controller.cc index c0d9c3efda892..e457f5986e1e7 100644 --- a/runtime/runtime_controller.cc +++ b/runtime/runtime_controller.cc @@ -22,6 +22,7 @@ RuntimeController::RuntimeController( TaskRunners p_task_runners, fml::WeakPtr p_snapshot_delegate, fml::WeakPtr p_io_manager, + fml::WeakPtr p_image_decoder, std::string p_advisory_script_uri, std::string p_advisory_script_entrypoint, std::function p_idle_notification_callback, @@ -34,6 +35,7 @@ RuntimeController::RuntimeController( std::move(p_task_runners), std::move(p_snapshot_delegate), std::move(p_io_manager), + std::move(p_image_decoder), std::move(p_advisory_script_uri), std::move(p_advisory_script_entrypoint), p_idle_notification_callback, @@ -49,6 +51,7 @@ RuntimeController::RuntimeController( TaskRunners p_task_runners, fml::WeakPtr p_snapshot_delegate, fml::WeakPtr p_io_manager, + fml::WeakPtr p_image_decoder, std::string p_advisory_script_uri, std::string p_advisory_script_entrypoint, std::function idle_notification_callback, @@ -62,6 +65,7 @@ RuntimeController::RuntimeController( task_runners_(p_task_runners), snapshot_delegate_(p_snapshot_delegate), io_manager_(p_io_manager), + image_decoder_(p_image_decoder), advisory_script_uri_(p_advisory_script_uri), advisory_script_entrypoint_(p_advisory_script_entrypoint), idle_notification_callback_(idle_notification_callback), @@ -79,6 +83,7 @@ RuntimeController::RuntimeController( std::make_unique(this), // snapshot_delegate_, // io_manager_, // + image_decoder_, // p_advisory_script_uri, // p_advisory_script_entrypoint, // nullptr, // @@ -139,6 +144,7 @@ std::unique_ptr RuntimeController::Clone() const { task_runners_, // snapshot_delegate_, // io_manager_, // + image_decoder_, // advisory_script_uri_, // advisory_script_entrypoint_, // idle_notification_callback_, // diff --git a/runtime/runtime_controller.h b/runtime/runtime_controller.h index 5a125013ea484..c3b3e35fa064c 100644 --- a/runtime/runtime_controller.h +++ b/runtime/runtime_controller.h @@ -35,6 +35,7 @@ class RuntimeController final : public WindowClient { TaskRunners task_runners, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr iamge_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, std::function idle_notification_callback, @@ -130,6 +131,7 @@ class RuntimeController final : public WindowClient { TaskRunners task_runners_; fml::WeakPtr snapshot_delegate_; fml::WeakPtr io_manager_; + fml::WeakPtr image_decoder_; std::string advisory_script_uri_; std::string advisory_script_entrypoint_; std::function idle_notification_callback_; @@ -146,6 +148,7 @@ class RuntimeController final : public WindowClient { TaskRunners task_runners, fml::WeakPtr snapshot_delegate, fml::WeakPtr io_manager, + fml::WeakPtr image_decoder, std::string advisory_script_uri, std::string advisory_script_entrypoint, std::function idle_notification_callback, diff --git a/shell/common/engine.cc b/shell/common/engine.cc index 5147789d7067c..d2c752a4747df 100644 --- a/shell/common/engine.cc +++ b/shell/common/engine.cc @@ -49,6 +49,9 @@ Engine::Engine(Delegate& delegate, animator_(std::move(animator)), activity_running_(false), have_surface_(false), + image_decoder_(task_runners, + vm.GetConcurrentWorkerTaskRunner(), + io_manager), weak_factory_(this) { // Runtime controller is initialized here because it takes a reference to this // object as its delegate. The delegate may be called in the constructor and @@ -61,6 +64,7 @@ Engine::Engine(Delegate& delegate, std::move(task_runners), // task runners std::move(snapshot_delegate), // snapshot delegate std::move(io_manager), // io manager + image_decoder_.GetWeakPtr(), // image decoder settings_.advisory_script_uri, // advisory script uri settings_.advisory_script_entrypoint, // advisory script entrypoint settings_.idle_notification_callback, // idle notification callback diff --git a/shell/common/engine.h b/shell/common/engine.h index 293d78c1a7178..95739c9b2bf61 100644 --- a/shell/common/engine.h +++ b/shell/common/engine.h @@ -12,6 +12,7 @@ #include "flutter/common/task_runners.h" #include "flutter/fml/macros.h" #include "flutter/fml/memory/weak_ptr.h" +#include "flutter/lib/ui/painting/image_decoder.h" #include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/lib/ui/semantics/semantics_node.h" #include "flutter/lib/ui/snapshot_delegate.h" @@ -138,6 +139,7 @@ class Engine final : public RuntimeDelegate { bool activity_running_; bool have_surface_; FontCollection font_collection_; + ImageDecoder image_decoder_; fml::WeakPtrFactory weak_factory_; // |RuntimeDelegate| diff --git a/shell/common/shell_io_manager.cc b/shell/common/shell_io_manager.cc index f1155c2e883a0..76db8505c9be1 100644 --- a/shell/common/shell_io_manager.cc +++ b/shell/common/shell_io_manager.cc @@ -52,7 +52,7 @@ ShellIOManager::ShellIOManager( : nullptr), unref_queue_(fml::MakeRefCounted( std::move(unref_queue_task_runner), - fml::TimeDelta::FromMilliseconds(250))), + fml::TimeDelta::FromMilliseconds(8))), weak_factory_(this) { if (!resource_context_) { #ifndef OS_FUCHSIA @@ -69,12 +69,6 @@ ShellIOManager::~ShellIOManager() { unref_queue_->Drain(); } -fml::WeakPtr ShellIOManager::GetResourceContext() const { - return resource_context_weak_factory_ - ? resource_context_weak_factory_->GetWeakPtr() - : fml::WeakPtr(); -} - void ShellIOManager::NotifyResourceContextAvailable( sk_sp resource_context) { // The resource context needs to survive as long as we have Dart objects @@ -93,11 +87,25 @@ void ShellIOManager::UpdateResourceContext(sk_sp resource_context) { : nullptr; } +fml::WeakPtr ShellIOManager::GetWeakPtr() { + return weak_factory_.GetWeakPtr(); +} + +// |IOManager| +fml::WeakPtr ShellIOManager::GetResourceContext() const { + return resource_context_weak_factory_ + ? resource_context_weak_factory_->GetWeakPtr() + : fml::WeakPtr(); +} + +// |IOManager| fml::RefPtr ShellIOManager::GetSkiaUnrefQueue() const { return unref_queue_; } -fml::WeakPtr ShellIOManager::GetWeakPtr() { +// |IOManager| +fml::WeakPtr ShellIOManager::GetWeakIOManager() const { return weak_factory_.GetWeakPtr(); } + } // namespace flutter diff --git a/shell/common/shell_io_manager.h b/shell/common/shell_io_manager.h index 11efd343437b7..066937f53bdaa 100644 --- a/shell/common/shell_io_manager.h +++ b/shell/common/shell_io_manager.h @@ -29,8 +29,6 @@ class ShellIOManager final : public IOManager { ~ShellIOManager() override; - fml::WeakPtr GetResourceContext() const override; - // This method should be called when a resource_context first becomes // available. It is safe to call multiple times, and will only update // the held resource context if it has not already been set. @@ -42,10 +40,17 @@ class ShellIOManager final : public IOManager { // resource context, but may be called if the Dart VM is restarted. void UpdateResourceContext(sk_sp resource_context); - fml::RefPtr GetSkiaUnrefQueue() const override; - fml::WeakPtr GetWeakPtr(); + // |IOManager| + fml::WeakPtr GetWeakIOManager() const override; + + // |IOManager| + fml::WeakPtr GetResourceContext() const override; + + // |IOManager| + fml::RefPtr GetSkiaUnrefQueue() const override; + private: // Resource context management. sk_sp resource_context_; diff --git a/shell/platform/embedder/BUILD.gn b/shell/platform/embedder/BUILD.gn index 1173a4f778715..c875d981162f4 100644 --- a/shell/platform/embedder/BUILD.gn +++ b/shell/platform/embedder/BUILD.gn @@ -67,10 +67,7 @@ if (current_toolchain == host_toolchain) { executable("embedder_unittests") { testonly = true - configs += [ - "$flutter_root:export_dynamic_symbols", - "//third_party/swiftshader_flutter:swiftshader_config", - ] + configs += [ "$flutter_root:export_dynamic_symbols" ] include_dirs = [ "." ] @@ -82,8 +79,6 @@ if (current_toolchain == host_toolchain) { "tests/embedder_context.h", "tests/embedder_test.cc", "tests/embedder_test.h", - "tests/embedder_test_gl_surface.cc", - "tests/embedder_test_gl_surface.h", "tests/embedder_unittests.cc", ] @@ -93,8 +88,8 @@ if (current_toolchain == host_toolchain) { "$flutter_root/lib/ui", "$flutter_root/runtime", "$flutter_root/testing:dart", + "$flutter_root/testing:opengl", "//third_party/skia", - "//third_party/swiftshader_flutter:swiftshader", "//third_party/tonic", ] } diff --git a/shell/platform/embedder/tests/embedder_context.cc b/shell/platform/embedder/tests/embedder_context.cc index ae759d667315f..5b8297547c66b 100644 --- a/shell/platform/embedder/tests/embedder_context.cc +++ b/shell/platform/embedder/tests/embedder_context.cc @@ -124,7 +124,7 @@ EmbedderContext::GetUpdateSemanticsCustomActionCallbackHook() { } void EmbedderContext::SetupOpenGLSurface() { - gl_surface_ = std::make_unique(); + gl_surface_ = std::make_unique(); } bool EmbedderContext::GLMakeCurrent() { diff --git a/shell/platform/embedder/tests/embedder_context.h b/shell/platform/embedder/tests/embedder_context.h index 7e9754e096233..02f617411a220 100644 --- a/shell/platform/embedder/tests/embedder_context.h +++ b/shell/platform/embedder/tests/embedder_context.h @@ -14,8 +14,8 @@ #include "flutter/fml/macros.h" #include "flutter/fml/mapping.h" #include "flutter/shell/platform/embedder/embedder.h" -#include "flutter/shell/platform/embedder/tests/embedder_test_gl_surface.h" #include "flutter/testing/test_dart_native_resolver.h" +#include "flutter/testing/test_gl_surface.h" namespace flutter { namespace testing { @@ -65,8 +65,8 @@ class EmbedderContext { std::shared_ptr native_resolver_; SemanticsNodeCallback update_semantics_node_callback_; SemanticsActionCallback update_semantics_custom_action_callback_; - std::unique_ptr gl_surface_; // lazy std::function platform_message_callback_; + std::unique_ptr gl_surface_; static VoidCallback GetIsolateCreateCallbackHook(); diff --git a/testing/BUILD.gn b/testing/BUILD.gn index f1cf5b7afebbd..82107d7f4e075 100644 --- a/testing/BUILD.gn +++ b/testing/BUILD.gn @@ -45,3 +45,25 @@ source_set("dart") { "//third_party/tonic", ] } + +if (current_toolchain == host_toolchain) { + source_set("opengl") { + testonly = true + + configs += [ "//third_party/swiftshader_flutter:swiftshader_config" ] + + sources = [ + "$flutter_root/testing/test_gl_surface.cc", + "$flutter_root/testing/test_gl_surface.h", + ] + + deps = [ + "//$flutter_root/fml", + "//third_party/swiftshader_flutter:swiftshader", + ] + + public_deps = [ + "//third_party/skia", + ] + } +} diff --git a/testing/run_tests.sh b/testing/run_tests.sh index 36946316ced09..68f353de9893c 100755 --- a/testing/run_tests.sh +++ b/testing/run_tests.sh @@ -54,6 +54,9 @@ echo "Running client_wrapper_glfw_unittests..." echo "Running txt_unittests..." "$HOST_DIR/txt_unittests" --font-directory="$BUILDROOT_DIR/flutter/third_party/txt/third_party/fonts" +echo "Running ui_unittests..." +"$HOST_DIR/ui_unittests" + echo "Running txt_benchmarks..." "$HOST_DIR/txt_benchmarks" --font-directory="$BUILDROOT_DIR/flutter/third_party/txt/third_party/fonts" diff --git a/shell/platform/embedder/tests/embedder_test_gl_surface.cc b/testing/test_gl_surface.cc similarity index 81% rename from shell/platform/embedder/tests/embedder_test_gl_surface.cc rename to testing/test_gl_surface.cc index 54c2d6e6ff961..ec8394078cef9 100644 --- a/shell/platform/embedder/tests/embedder_test_gl_surface.cc +++ b/testing/test_gl_surface.cc @@ -2,16 +2,19 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "flutter/shell/platform/embedder/tests/embedder_test_gl_surface.h" +#include "flutter/testing/test_gl_surface.h" #include +#include #include #include #include "flutter/fml/logging.h" +#include "third_party/skia/include/gpu/gl/GrGLAssembleInterface.h" namespace flutter { +namespace testing { static std::string GetEGLError() { std::stringstream stream; @@ -74,7 +77,7 @@ static std::string GetEGLError() { return stream.str(); } -EmbedderTestGLSurface::EmbedderTestGLSurface() { +TestGLSurface::TestGLSurface() { display_ = ::eglGetDisplay(EGL_DEFAULT_DISPLAY); FML_CHECK(display_ != EGL_NO_DISPLAY); @@ -151,7 +154,7 @@ EmbedderTestGLSurface::EmbedderTestGLSurface() { } } -EmbedderTestGLSurface::~EmbedderTestGLSurface() { +TestGLSurface::~TestGLSurface() { auto result = ::eglDestroyContext(display_, onscreen_context_); FML_CHECK(result == EGL_TRUE) << GetEGLError(); @@ -168,7 +171,7 @@ EmbedderTestGLSurface::~EmbedderTestGLSurface() { FML_CHECK(result == EGL_TRUE); } -bool EmbedderTestGLSurface::MakeCurrent() { +bool TestGLSurface::MakeCurrent() { auto result = ::eglMakeCurrent(display_, onscreen_surface_, onscreen_surface_, onscreen_context_); @@ -179,7 +182,7 @@ bool EmbedderTestGLSurface::MakeCurrent() { return result == EGL_TRUE; } -bool EmbedderTestGLSurface::ClearCurrent() { +bool TestGLSurface::ClearCurrent() { auto result = ::eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); @@ -190,7 +193,7 @@ bool EmbedderTestGLSurface::ClearCurrent() { return result == EGL_TRUE; } -bool EmbedderTestGLSurface::Present() { +bool TestGLSurface::Present() { auto result = ::eglSwapBuffers(display_, onscreen_surface_); if (result == EGL_FALSE) { @@ -200,12 +203,12 @@ bool EmbedderTestGLSurface::Present() { return result == EGL_TRUE; } -uint32_t EmbedderTestGLSurface::GetFramebuffer() { +uint32_t TestGLSurface::GetFramebuffer() { // Return FBO0 return 0; } -bool EmbedderTestGLSurface::MakeResourceCurrent() { +bool TestGLSurface::MakeResourceCurrent() { auto result = ::eglMakeCurrent(display_, offscreen_surface_, offscreen_surface_, offscreen_context_); @@ -217,7 +220,10 @@ bool EmbedderTestGLSurface::MakeResourceCurrent() { return result == EGL_TRUE; } -void* EmbedderTestGLSurface::GetProcAddress(const char* name) { +void* TestGLSurface::GetProcAddress(const char* name) { + if (name == nullptr) { + return nullptr; + } auto symbol = ::eglGetProcAddress(name); if (symbol == NULL) { FML_LOG(ERROR) << "Could not fetch symbol for name: " << name; @@ -225,4 +231,40 @@ void* EmbedderTestGLSurface::GetProcAddress(const char* name) { return reinterpret_cast(symbol); } +sk_sp TestGLSurface::CreateContext() { + if (!MakeCurrent()) { + return nullptr; + } + + auto get_string = + reinterpret_cast(GetProcAddress("glGetString")); + + if (!get_string) { + return nullptr; + } + + auto c_version = reinterpret_cast(get_string(GL_VERSION)); + + if (c_version == NULL) { + return nullptr; + } + + GrGLGetProc get_proc = [](void* context, const char name[]) -> GrGLFuncPtr { + return reinterpret_cast( + reinterpret_cast(context)->GetProcAddress(name)); + }; + + std::string version(c_version); + auto interface = version.find("OpenGL ES") == std::string::npos + ? GrGLMakeAssembledGLInterface(this, get_proc) + : GrGLMakeAssembledGLESInterface(this, get_proc); + + if (!interface) { + return nullptr; + } + + return GrContext::MakeGL(interface); +} + +} // namespace testing } // namespace flutter diff --git a/shell/platform/embedder/tests/embedder_test_gl_surface.h b/testing/test_gl_surface.h similarity index 58% rename from shell/platform/embedder/tests/embedder_test_gl_surface.h rename to testing/test_gl_surface.h index 0d7da604310ff..3357ce56fe1bc 100644 --- a/shell/platform/embedder/tests/embedder_test_gl_surface.h +++ b/testing/test_gl_surface.h @@ -2,19 +2,22 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef FLUTTER_SHELL_PLATFORM_EMBEDDER_TESTS_EMBEDDER_TEST_GL_SURFACE_H_ -#define FLUTTER_SHELL_PLATFORM_EMBEDDER_TESTS_EMBEDDER_TEST_GL_SURFACE_H_ +#ifndef FLUTTER_TESTING_TEST_GL_SURFACE_H_ +#define FLUTTER_TESTING_TEST_GL_SURFACE_H_ + +#include #include "flutter/fml/macros.h" -#include "flutter/shell/platform/embedder/embedder.h" +#include "third_party/skia/include/gpu/GrContext.h" namespace flutter { +namespace testing { -class EmbedderTestGLSurface { +class TestGLSurface { public: - EmbedderTestGLSurface(); + TestGLSurface(); - ~EmbedderTestGLSurface(); + ~TestGLSurface(); bool MakeCurrent(); @@ -28,11 +31,13 @@ class EmbedderTestGLSurface { void* GetProcAddress(const char* name); + sk_sp CreateContext(); + private: // Importing the EGL.h pulls in platform headers which are problematic - // (especially X11 which #defineds types like Bool). Any TUs importing this - // header then become susceptible to failures because of platform specific - // craziness. Don't expose EGL internals via this header. + // (especially X11 which #defineds types like Bool). Any TUs importing + // this header then become susceptible to failures because of platform + // specific craziness. Don't expose EGL internals via this header. using EGLDisplay = void*; using EGLContext = void*; using EGLSurface = void*; @@ -43,9 +48,10 @@ class EmbedderTestGLSurface { EGLSurface onscreen_surface_; EGLSurface offscreen_surface_; - FML_DISALLOW_COPY_AND_ASSIGN(EmbedderTestGLSurface); + FML_DISALLOW_COPY_AND_ASSIGN(TestGLSurface); }; +} // namespace testing } // namespace flutter -#endif // FLUTTER_SHELL_PLATFORM_EMBEDDER_TESTS_EMBEDDER_TEST_GL_SURFACE_H_ +#endif // FLUTTER_TESTING_TEST_GL_SURFACE_H_ diff --git a/testing/testing.cc b/testing/testing.cc index 2e2225a568036..270d6f00b7e93 100644 --- a/testing/testing.cc +++ b/testing/testing.cc @@ -4,6 +4,8 @@ #include "testing.h" +#include "flutter/fml/file.h" + namespace flutter { namespace testing { @@ -11,5 +13,36 @@ std::string GetCurrentTestName() { return ::testing::UnitTest::GetInstance()->current_test_info()->name(); } +fml::UniqueFD OpenFixture(std::string fixture_name) { + if (fixture_name.size() == 0) { + FML_LOG(ERROR) << "Invalid fixture name."; + return {}; + } + + auto fixtures_directory = + OpenDirectory(GetFixturesPath(), // path + false, // create + fml::FilePermission::kRead // permission + ); + + if (!fixtures_directory.is_valid()) { + FML_LOG(ERROR) << "Could not open fixtures directory."; + return {}; + } + + auto fixture_fd = fml::OpenFile(fixtures_directory, // base directory + fixture_name.c_str(), // path + false, // create + fml::FilePermission::kRead // permission + ); + if (!fixture_fd.is_valid()) { + FML_LOG(ERROR) << "Could not open fixture for path: " << fixture_name + << "."; + return {}; + } + + return fixture_fd; +} + } // namespace testing } // namespace flutter diff --git a/testing/testing.h b/testing/testing.h index 00fe3806488dc..fc4fe6a4d8e66 100644 --- a/testing/testing.h +++ b/testing/testing.h @@ -7,6 +7,7 @@ #include +#include "flutter/fml/file.h" #include "gtest/gtest.h" namespace flutter { @@ -17,6 +18,8 @@ namespace testing { // error. const char* GetFixturesPath(); +fml::UniqueFD OpenFixture(std::string fixture_name); + std::string GetCurrentTestName(); } // namespace testing diff --git a/testing/thread_test.cc b/testing/thread_test.cc index 986afd4e62fbb..deb371e63b472 100644 --- a/testing/thread_test.cc +++ b/testing/thread_test.cc @@ -23,6 +23,7 @@ void ThreadTest::TearDown() { thread_task_runner_ = nullptr; thread_ = nullptr; current_task_runner_ = nullptr; + extra_threads_.clear(); } fml::RefPtr ThreadTest::GetCurrentTaskRunner() { @@ -33,5 +34,12 @@ fml::RefPtr ThreadTest::GetThreadTaskRunner() { return thread_task_runner_; } +fml::RefPtr ThreadTest::CreateNewThread(std::string name) { + auto thread = std::make_unique(name); + auto runner = thread->GetTaskRunner(); + extra_threads_.emplace_back(std::move(thread)); + return runner; +} + } // namespace testing } // namespace flutter diff --git a/testing/thread_test.h b/testing/thread_test.h index 6d168b0d0c01a..270628b114276 100644 --- a/testing/thread_test.h +++ b/testing/thread_test.h @@ -6,6 +6,7 @@ #define FLUTTER_TESTING_THREAD_TEST_H_ #include +#include #include "flutter/fml/macros.h" #include "flutter/fml/message_loop.h" @@ -22,6 +23,8 @@ class ThreadTest : public ::testing::Test { fml::RefPtr GetThreadTaskRunner(); + fml::RefPtr CreateNewThread(std::string name = ""); + protected: // |testing::Test| void SetUp() override; @@ -33,6 +36,7 @@ class ThreadTest : public ::testing::Test { std::unique_ptr thread_; fml::RefPtr thread_task_runner_; fml::RefPtr current_task_runner_; + std::vector> extra_threads_; }; } // namespace testing