[Impeller Scene] Compute joint transforms and apply them to skinned meshes

ci/licenses_golden/licenses_flutter

@@ -1647,6 +1647,8 @@ ORIGIN: ../../../flutter/impeller/scene/scene_encoder.h + ../../../flutter/LICEN
 ORIGIN: ../../../flutter/impeller/scene/shaders/skinned.vert + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/scene/shaders/unlit.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/scene/shaders/unskinned.vert + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/scene/skin.cc + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/scene/skin.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE
@@ -4116,6 +4118,8 @@ FILE: ../../../flutter/impeller/scene/scene_encoder.h
 FILE: ../../../flutter/impeller/scene/shaders/skinned.vert
 FILE: ../../../flutter/impeller/scene/shaders/unlit.frag
 FILE: ../../../flutter/impeller/scene/shaders/unskinned.vert
+FILE: ../../../flutter/impeller/scene/skin.cc
+FILE: ../../../flutter/impeller/scene/skin.h
 FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc
 FILE: ../../../flutter/impeller/tessellator/c/tessellator.h
 FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart

impeller/renderer/backend/gles/texture_gles.cc

@@ -107,6 +107,16 @@ struct TexImage2DData {
         external_format = GL_RGBA;
         type = GL_UNSIGNED_BYTE;
         break;
+      case PixelFormat::kR32G32B32A32Float:
+        internal_format = GL_RGBA;
+        external_format = GL_RGBA;
+        type = GL_FLOAT;
+        break;
+      case PixelFormat::kR16G16B16A16Float:
+        internal_format = GL_RGBA;
+        external_format = GL_RGBA;
+        type = GL_HALF_FLOAT;
+        break;
       case PixelFormat::kUnknown:
       case PixelFormat::kS8UInt:
       case PixelFormat::kD32FloatS8UInt:
@@ -136,6 +146,20 @@ struct TexImage2DData {
         data = std::move(mapping);
         break;
       }
+      case PixelFormat::kR32G32B32A32Float: {
+        internal_format = GL_RGBA;
+        external_format = GL_RGBA;
+        type = GL_FLOAT;
+        data = std::move(mapping);
+        break;
+      }
+      case PixelFormat::kR16G16B16A16Float: {
+        internal_format = GL_RGBA;
+        external_format = GL_RGBA;
+        type = GL_HALF_FLOAT;
+        data = std::move(mapping);
+        break;
+      }
       case PixelFormat::kR8G8B8A8UNormIntSRGB:
       case PixelFormat::kB8G8R8A8UNormInt:
       case PixelFormat::kB8G8R8A8UNormIntSRGB:
@@ -274,6 +298,10 @@ static std::optional<GLenum> ToRenderBufferFormat(PixelFormat format) {
     case PixelFormat::kB8G8R8A8UNormInt:
     case PixelFormat::kR8G8B8A8UNormInt:
       return GL_RGBA4;
+    case PixelFormat::kR32G32B32A32Float:
+      return GL_RGBA32F;
+    case PixelFormat::kR16G16B16A16Float:
+      return GL_RGBA16F;
     case PixelFormat::kS8UInt:
       return GL_STENCIL_INDEX8;
     case PixelFormat::kD32FloatS8UInt:

impeller/renderer/backend/metal/formats_mtl.h

@@ -29,6 +29,10 @@ constexpr PixelFormat FromMTLPixelFormat(MTLPixelFormat format) {
       return PixelFormat::kR8G8B8A8UNormInt;
     case MTLPixelFormatRGBA8Unorm_sRGB:
       return PixelFormat::kR8G8B8A8UNormIntSRGB;
+    case MTLPixelFormatRGBA32Float:
+      return PixelFormat::kR32G32B32A32Float;
+    case MTLPixelFormatRGBA16Float:
+      return PixelFormat::kR16G16B16A16Float;
     case MTLPixelFormatStencil8:
       return PixelFormat::kS8UInt;
     case MTLPixelFormatDepth32Float_Stencil8:
@@ -57,6 +61,10 @@ constexpr MTLPixelFormat ToMTLPixelFormat(PixelFormat format) {
       return MTLPixelFormatRGBA8Unorm;
     case PixelFormat::kR8G8B8A8UNormIntSRGB:
       return MTLPixelFormatRGBA8Unorm_sRGB;
+    case PixelFormat::kR32G32B32A32Float:
+      return MTLPixelFormatRGBA32Float;
+    case PixelFormat::kR16G16B16A16Float:
+      return MTLPixelFormatRGBA16Float;
     case PixelFormat::kS8UInt:
       return MTLPixelFormatStencil8;
     case PixelFormat::kD32FloatS8UInt:

impeller/renderer/backend/vulkan/formats_vk.h

@@ -148,6 +148,10 @@ constexpr vk::Format ToVKImageFormat(PixelFormat format) {
       return vk::Format::eB8G8R8A8Unorm;
     case PixelFormat::kB8G8R8A8UNormIntSRGB:
       return vk::Format::eB8G8R8A8Srgb;
+    case PixelFormat::kR32G32B32A32Float:
+      return vk::Format::eR32G32B32A32Sfloat;
+    case PixelFormat::kR16G16B16A16Float:
+      return vk::Format::eR16G16B16A16Sfloat;
     case PixelFormat::kS8UInt:
       return vk::Format::eS8Uint;
     case PixelFormat::kD32FloatS8UInt:
@@ -178,6 +182,12 @@ constexpr PixelFormat ToPixelFormat(vk::Format format) {
     case vk::Format::eB8G8R8A8Srgb:
       return PixelFormat::kB8G8R8A8UNormIntSRGB;
 
+    case vk::Format::eR32G32B32A32Sfloat:
+      return PixelFormat::kR32G32B32A32Float;
+
+    case vk::Format::eR16G16B16A16Sfloat:
+      return PixelFormat::kR16G16B16A16Float;
+
     case vk::Format::eS8Uint:
       return PixelFormat::kS8UInt;
 

impeller/renderer/formats.h

@@ -87,6 +87,8 @@ enum class PixelFormat {
   kR8G8B8A8UNormIntSRGB,
   kB8G8R8A8UNormInt,
   kB8G8R8A8UNormIntSRGB,
+  kR32G32B32A32Float,
+  kR16G16B16A16Float,
 
   // Depth and stencil formats.
   kS8UInt,
@@ -290,6 +292,10 @@ constexpr size_t BytesPerPixelForPixelFormat(PixelFormat format) {
       return 4u;
     case PixelFormat::kD32FloatS8UInt:
       return 5u;
+    case PixelFormat::kR16G16B16A16Float:
+      return 8u;
+    case PixelFormat::kR32G32B32A32Float:
+      return 16u;
   }
   return 0u;
 }

impeller/scene/BUILD.gn

@@ -31,6 +31,8 @@ impeller_component("scene") {
     "scene_context.h",
     "scene_encoder.cc",
     "scene_encoder.h",
+    "skin.cc",
+    "skin.h",
   ]
 
   public_deps = [

impeller/scene/animation/animation_player.cc

@@ -54,7 +54,7 @@ void AnimationPlayer::Update() {
 
 void AnimationPlayer::Reset() {
   for (auto& [node, transform] : default_target_transforms_) {
-    node->SetLocalTransform(transform);
+    node->SetLocalTransform(Matrix());
   }
 }
 

impeller/scene/animation/property_resolver.cc

@@ -89,8 +89,8 @@ void TranslationTimelineResolver::Apply(Node& target,
   if (key.lerp < 1) {
     value = values_[key.index - 1].Lerp(value, key.lerp);
   }
-  target.SetLocalTransform(Matrix::MakeTranslation(value * weight) *
-                           target.GetLocalTransform());
+  target.SetLocalTransform(target.GetLocalTransform() *
+                           Matrix::MakeTranslation(value * weight));
 }
 
 RotationTimelineResolver::RotationTimelineResolver() = default;
@@ -108,8 +108,8 @@ void RotationTimelineResolver::Apply(Node& target,
   if (key.lerp < 1) {
     value = values_[key.index - 1].Slerp(value, key.lerp);
   }
-  target.SetLocalTransform(Matrix::MakeRotation(value * weight) *
-                           target.GetLocalTransform());
+  target.SetLocalTransform(target.GetLocalTransform() *
+                           Matrix::MakeRotation(value * weight));
 }
 
 ScaleTimelineResolver::ScaleTimelineResolver() = default;
@@ -125,8 +125,8 @@ void ScaleTimelineResolver::Apply(Node& target, SecondsF time, Scalar weight) {
   if (key.lerp < 1) {
     value = values_[key.index - 1].Lerp(value, key.lerp);
   }
-  target.SetLocalTransform(Matrix::MakeScale(value * weight) *
-                           target.GetLocalTransform());
+  target.SetLocalTransform(target.GetLocalTransform() *
+                           Matrix::MakeScale(value * weight));
 }
 
 }  // namespace scene

impeller/scene/geometry.cc

@@ -12,6 +12,8 @@
 #include "impeller/geometry/vector.h"
 #include "impeller/renderer/device_buffer_descriptor.h"
 #include "impeller/renderer/formats.h"
+#include "impeller/renderer/sampler_descriptor.h"
+#include "impeller/renderer/sampler_library.h"
 #include "impeller/renderer/vertex_buffer.h"
 #include "impeller/renderer/vertex_buffer_builder.h"
 #include "impeller/scene/importer/scene_flatbuffers.h"
@@ -117,6 +119,8 @@ std::shared_ptr<Geometry> Geometry::MakeFromFlatbuffer(
   return MakeVertexBuffer(std::move(vertex_buffer), is_skinned);
 }
 
+void Geometry::SetJointsTexture(const std::shared_ptr<Texture>& texture) {}
+
 //------------------------------------------------------------------------------
 /// CuboidGeometry
 ///
@@ -239,10 +243,31 @@ void SkinnedVertexBufferGeometry::BindToCommand(
   command.BindVertices(
       GetVertexBuffer(*scene_context.GetContext()->GetResourceAllocator()));
 
+  SamplerDescriptor sampler_desc;
+  sampler_desc.min_filter = MinMagFilter::kNearest;
+  sampler_desc.mag_filter = MinMagFilter::kNearest;
+  sampler_desc.mip_filter = MipFilter::kNone;
+  sampler_desc.width_address_mode = SamplerAddressMode::kRepeat;
+  sampler_desc.label = "NN Repeat";
+
+  SkinnedVertexShader::BindJointsTexture(
+      command,
+      joints_texture_ ? joints_texture_ : scene_context.GetPlaceholderTexture(),
+      scene_context.GetContext()->GetSamplerLibrary()->GetSampler(
+          sampler_desc));
+
   SkinnedVertexShader::VertInfo info;
   info.mvp = transform;
+  info.enable_skinning = joints_texture_ ? 1 : 0;
+  info.joint_texture_size =
+      joints_texture_ ? joints_texture_->GetSize().width : 1;
   SkinnedVertexShader::BindVertInfo(command, buffer.EmplaceUniform(info));
 }
 
+// |Geometry|
+void SkinnedVertexBufferGeometry::SetJointsTexture(
+    const std::shared_ptr<Texture>& texture) {
+  joints_texture_ = texture;
+}
 }  // namespace scene
 }  // namespace impeller

impeller/scene/geometry.h

@@ -45,6 +45,8 @@ class Geometry {
                              HostBuffer& buffer,
                              const Matrix& transform,
                              Command& command) const = 0;
+
+  virtual void SetJointsTexture(const std::shared_ptr<Texture>& texture);
 };
 
 class CuboidGeometry final : public Geometry {
@@ -119,8 +121,12 @@ class SkinnedVertexBufferGeometry final : public Geometry {
                      const Matrix& transform,
                      Command& command) const override;
 
+  // |Geometry|
+  void SetJointsTexture(const std::shared_ptr<Texture>& texture) override;
+
  private:
   VertexBuffer vertex_buffer_;
+  std::shared_ptr<Texture> joints_texture_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(SkinnedVertexBufferGeometry);
 };

impeller/scene/importer/importer_gltf.cc

@@ -8,6 +8,7 @@
 #include <cstring>
 #include <functional>
 #include <iostream>
+#include <iterator>
 #include <memory>
 #include <vector>
 
@@ -199,22 +200,22 @@ static void ProcessNode(const tinygltf::Model& gltf,
   ///
 
   Matrix transform;
-  if (in_node.translation.size() == 3) {
-    transform = transform * Matrix::MakeTranslation(
-                                {static_cast<Scalar>(in_node.translation[0]),
-                                 static_cast<Scalar>(in_node.translation[1]),
-                                 static_cast<Scalar>(in_node.translation[2])});
+  if (in_node.scale.size() == 3) {
+    transform =
+        transform * Matrix::MakeScale({static_cast<Scalar>(in_node.scale[0]),
+                                       static_cast<Scalar>(in_node.scale[1]),
+                                       static_cast<Scalar>(in_node.scale[2])});
   }
   if (in_node.rotation.size() == 4) {
     transform = transform * Matrix::MakeRotation(Quaternion(
                                 in_node.rotation[0], in_node.rotation[1],
                                 in_node.rotation[2], in_node.rotation[3]));
   }
-  if (in_node.scale.size() == 3) {
-    transform =
-        transform * Matrix::MakeScale({static_cast<Scalar>(in_node.scale[0]),
-                                       static_cast<Scalar>(in_node.scale[1]),
-                                       static_cast<Scalar>(in_node.scale[2])});
+  if (in_node.translation.size() == 3) {
+    transform = transform * Matrix::MakeTranslation(
+                                {static_cast<Scalar>(in_node.translation[0]),
+                                 static_cast<Scalar>(in_node.translation[1]),
+                                 static_cast<Scalar>(in_node.translation[2])});
   }
   if (in_node.matrix.size() == 16) {
     if (!transform.IsIdentity()) {
@@ -317,11 +318,14 @@ static void ProcessAnimation(const tinygltf::Model& gltf,
                              fb::AnimationT& out_animation) {
   out_animation.name = in_animation.name;
 
-  std::vector<std::unique_ptr<impeller::fb::ChannelT>> channels;
+  // std::vector<impeller::fb::ChannelT> channels;
+  std::vector<impeller::fb::ChannelT> translation_channels;
+  std::vector<impeller::fb::ChannelT> rotation_channels;
+  std::vector<impeller::fb::ChannelT> scale_channels;
   for (auto& in_channel : in_animation.channels) {
-    auto out_channel = std::make_unique<fb::ChannelT>();
+    auto out_channel = fb::ChannelT();
 
-    out_channel->node = in_channel.target_node;
+    out_channel.node = in_channel.target_node;
     auto& sampler = in_animation.samplers[in_channel.sampler];
 
     /// Keyframe times.
@@ -349,7 +353,7 @@ static void ProcessAnimation(const tinygltf::Model& gltf,
         const float* time_p = reinterpret_cast<const float*>(
             times_buffer.data.data() + times_bufferview.byteOffset +
             times_accessor.ByteStride(times_bufferview) * time_i);
-        out_channel->timeline.push_back(*time_p);
+        out_channel.timeline.push_back(*time_p);
       }
     }
 
@@ -387,7 +391,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf,
           keyframes.values.push_back(
               fb::Vec3(value_p[0], value_p[1], value_p[2]));
         }
-        out_channel->keyframes.Set(std::move(keyframes));
+        out_channel.keyframes.Set(std::move(keyframes));
+        translation_channels.push_back(std::move(out_channel));
       } else if (in_channel.target_path == "rotation") {
         if (values_accessor.type != TINYGLTF_TYPE_VEC4) {
           std::cerr << "Unexpected type \"" << values_accessor.type
@@ -404,7 +409,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf,
           keyframes.values.push_back(
               fb::Vec4(value_p[0], value_p[1], value_p[2], value_p[3]));
         }
-        out_channel->keyframes.Set(std::move(keyframes));
+        out_channel.keyframes.Set(std::move(keyframes));
+        rotation_channels.push_back(std::move(out_channel));
       } else if (in_channel.target_path == "scale") {
         if (values_accessor.type != TINYGLTF_TYPE_VEC3) {
           std::cerr << "Unexpected type \"" << values_accessor.type
@@ -421,15 +427,22 @@ static void ProcessAnimation(const tinygltf::Model& gltf,
           keyframes.values.push_back(
               fb::Vec3(value_p[0], value_p[1], value_p[2]));
         }
-        out_channel->keyframes.Set(std::move(keyframes));
+        out_channel.keyframes.Set(std::move(keyframes));
+        scale_channels.push_back(std::move(out_channel));
       } else {
         std::cerr << "Unsupported animation channel target path \""
                   << in_channel.target_path << "\". Skipping." << std::endl;
         continue;
       }
     }
+  }
 
-    channels.push_back(std::move(out_channel));
+  std::vector<std::unique_ptr<impeller::fb::ChannelT>> channels;
+  for (const auto& channel_list :
+       {translation_channels, rotation_channels, scale_channels}) {
+    for (const auto& channel : channel_list) {
+      channels.push_back(std::make_unique<fb::ChannelT>(channel));
+    }
   }
   out_animation.channels = std::move(channels);
 }
@@ -458,6 +471,9 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
   const tinygltf::Scene& scene = gltf.scenes[gltf.defaultScene];
   out_scene.children = scene.nodes;
 
+  out_scene.transform =
+      ToFBMatrixUniquePtr(Matrix::MakeScale(Vector3(1, 1, -1)));
+
   for (size_t texture_i = 0; texture_i < gltf.textures.size(); texture_i++) {
     auto texture = std::make_unique<fb::TextureT>();
     ProcessTexture(gltf, gltf.textures[texture_i], *texture);

impeller/scene/importer/importer_unittests.cc

@@ -74,6 +74,8 @@ TEST(ImporterTest, CanParseSkinnedGLTF) {
   // The skinned node contains both a skeleton and skinned mesh primitives that
   // reference bones in the skeleton.
   auto& skinned_node = scene.nodes[node->children[0]];
+  ASSERT_NE(skinned_node->skin, nullptr);
+
   ASSERT_EQ(skinned_node->mesh_primitives.size(), 2u);
   auto& bottom_triangle = *skinned_node->mesh_primitives[0];
   ASSERT_EQ(bottom_triangle.indices->count, 3u);

impeller/scene/importer/scene.fbs

@@ -188,6 +188,7 @@ table Node {
 
 table Scene {
   children: [int];  // Indices into `Scene`->`nodes`.
+  transform: Matrix;
   nodes: [Node];
   textures: [Texture];  // Textures may be reused across different materials.
   animations: [Animation];