diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 34c5ab34c30d1..02e8ce98e055c 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1647,6 +1647,8 @@ ORIGIN: ../../../flutter/impeller/scene/scene_encoder.h + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/scene/shaders/skinned.vert + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/shaders/unlit.frag + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/shaders/unskinned.vert + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/scene/skin.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/scene/skin.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE @@ -4116,6 +4118,8 @@ FILE: ../../../flutter/impeller/scene/scene_encoder.h FILE: ../../../flutter/impeller/scene/shaders/skinned.vert FILE: ../../../flutter/impeller/scene/shaders/unlit.frag FILE: ../../../flutter/impeller/scene/shaders/unskinned.vert +FILE: ../../../flutter/impeller/scene/skin.cc +FILE: ../../../flutter/impeller/scene/skin.h FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc FILE: ../../../flutter/impeller/tessellator/c/tessellator.h FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart diff --git a/impeller/fixtures/two_triangles.glb b/impeller/fixtures/two_triangles.glb index 4c7b8803bc927..0ecae588ca88c 100644 Binary files a/impeller/fixtures/two_triangles.glb and b/impeller/fixtures/two_triangles.glb differ diff --git a/impeller/renderer/backend/gles/texture_gles.cc b/impeller/renderer/backend/gles/texture_gles.cc index e9346b7168e55..191e6c9d652bc 100644 --- a/impeller/renderer/backend/gles/texture_gles.cc +++ b/impeller/renderer/backend/gles/texture_gles.cc @@ -107,6 +107,16 @@ struct TexImage2DData { external_format = GL_RGBA; type = GL_UNSIGNED_BYTE; break; + case PixelFormat::kR32G32B32A32Float: + internal_format = GL_RGBA; + external_format = GL_RGBA; + type = GL_FLOAT; + break; + case PixelFormat::kR16G16B16A16Float: + internal_format = GL_RGBA; + external_format = GL_RGBA; + type = GL_HALF_FLOAT; + break; case PixelFormat::kUnknown: case PixelFormat::kS8UInt: case PixelFormat::kD32FloatS8UInt: @@ -136,6 +146,20 @@ struct TexImage2DData { data = std::move(mapping); break; } + case PixelFormat::kR32G32B32A32Float: { + internal_format = GL_RGBA; + external_format = GL_RGBA; + type = GL_FLOAT; + data = std::move(mapping); + break; + } + case PixelFormat::kR16G16B16A16Float: { + internal_format = GL_RGBA; + external_format = GL_RGBA; + type = GL_HALF_FLOAT; + data = std::move(mapping); + break; + } case PixelFormat::kR8G8B8A8UNormIntSRGB: case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kB8G8R8A8UNormIntSRGB: @@ -274,6 +298,10 @@ static std::optional ToRenderBufferFormat(PixelFormat format) { case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kR8G8B8A8UNormInt: return GL_RGBA4; + case PixelFormat::kR32G32B32A32Float: + return GL_RGBA32F; + case PixelFormat::kR16G16B16A16Float: + return GL_RGBA16F; case PixelFormat::kS8UInt: return GL_STENCIL_INDEX8; case PixelFormat::kD32FloatS8UInt: diff --git a/impeller/renderer/backend/metal/formats_mtl.h b/impeller/renderer/backend/metal/formats_mtl.h index fc4956bec212d..056c9a73146ac 100644 --- a/impeller/renderer/backend/metal/formats_mtl.h +++ b/impeller/renderer/backend/metal/formats_mtl.h @@ -29,6 +29,10 @@ constexpr PixelFormat FromMTLPixelFormat(MTLPixelFormat format) { return PixelFormat::kR8G8B8A8UNormInt; case MTLPixelFormatRGBA8Unorm_sRGB: return PixelFormat::kR8G8B8A8UNormIntSRGB; + case MTLPixelFormatRGBA32Float: + return PixelFormat::kR32G32B32A32Float; + case MTLPixelFormatRGBA16Float: + return PixelFormat::kR16G16B16A16Float; case MTLPixelFormatStencil8: return PixelFormat::kS8UInt; case MTLPixelFormatDepth32Float_Stencil8: @@ -57,6 +61,10 @@ constexpr MTLPixelFormat ToMTLPixelFormat(PixelFormat format) { return MTLPixelFormatRGBA8Unorm; case PixelFormat::kR8G8B8A8UNormIntSRGB: return MTLPixelFormatRGBA8Unorm_sRGB; + case PixelFormat::kR32G32B32A32Float: + return MTLPixelFormatRGBA32Float; + case PixelFormat::kR16G16B16A16Float: + return MTLPixelFormatRGBA16Float; case PixelFormat::kS8UInt: return MTLPixelFormatStencil8; case PixelFormat::kD32FloatS8UInt: diff --git a/impeller/renderer/backend/vulkan/formats_vk.h b/impeller/renderer/backend/vulkan/formats_vk.h index f0d56741b3255..10441c3624e62 100644 --- a/impeller/renderer/backend/vulkan/formats_vk.h +++ b/impeller/renderer/backend/vulkan/formats_vk.h @@ -148,6 +148,10 @@ constexpr vk::Format ToVKImageFormat(PixelFormat format) { return vk::Format::eB8G8R8A8Unorm; case PixelFormat::kB8G8R8A8UNormIntSRGB: return vk::Format::eB8G8R8A8Srgb; + case PixelFormat::kR32G32B32A32Float: + return vk::Format::eR32G32B32A32Sfloat; + case PixelFormat::kR16G16B16A16Float: + return vk::Format::eR16G16B16A16Sfloat; case PixelFormat::kS8UInt: return vk::Format::eS8Uint; case PixelFormat::kD32FloatS8UInt: @@ -178,6 +182,12 @@ constexpr PixelFormat ToPixelFormat(vk::Format format) { case vk::Format::eB8G8R8A8Srgb: return PixelFormat::kB8G8R8A8UNormIntSRGB; + case vk::Format::eR32G32B32A32Sfloat: + return PixelFormat::kR32G32B32A32Float; + + case vk::Format::eR16G16B16A16Sfloat: + return PixelFormat::kR16G16B16A16Float; + case vk::Format::eS8Uint: return PixelFormat::kS8UInt; diff --git a/impeller/renderer/formats.h b/impeller/renderer/formats.h index a35378dbde1ec..12015586c992f 100644 --- a/impeller/renderer/formats.h +++ b/impeller/renderer/formats.h @@ -87,6 +87,8 @@ enum class PixelFormat { kR8G8B8A8UNormIntSRGB, kB8G8R8A8UNormInt, kB8G8R8A8UNormIntSRGB, + kR32G32B32A32Float, + kR16G16B16A16Float, // Depth and stencil formats. kS8UInt, @@ -290,6 +292,10 @@ constexpr size_t BytesPerPixelForPixelFormat(PixelFormat format) { return 4u; case PixelFormat::kD32FloatS8UInt: return 5u; + case PixelFormat::kR16G16B16A16Float: + return 8u; + case PixelFormat::kR32G32B32A32Float: + return 16u; } return 0u; } diff --git a/impeller/scene/BUILD.gn b/impeller/scene/BUILD.gn index 69c024f9e4232..dbd531cf8e17d 100644 --- a/impeller/scene/BUILD.gn +++ b/impeller/scene/BUILD.gn @@ -31,6 +31,8 @@ impeller_component("scene") { "scene_context.h", "scene_encoder.cc", "scene_encoder.h", + "skin.cc", + "skin.h", ] public_deps = [ diff --git a/impeller/scene/animation/animation_player.cc b/impeller/scene/animation/animation_player.cc index d2e9c353e03e1..01119fb521f61 100644 --- a/impeller/scene/animation/animation_player.cc +++ b/impeller/scene/animation/animation_player.cc @@ -54,7 +54,7 @@ void AnimationPlayer::Update() { void AnimationPlayer::Reset() { for (auto& [node, transform] : default_target_transforms_) { - node->SetLocalTransform(transform); + node->SetLocalTransform(Matrix()); } } diff --git a/impeller/scene/animation/property_resolver.cc b/impeller/scene/animation/property_resolver.cc index d4f5e3c7da830..d9d5db94e44e6 100644 --- a/impeller/scene/animation/property_resolver.cc +++ b/impeller/scene/animation/property_resolver.cc @@ -89,8 +89,8 @@ void TranslationTimelineResolver::Apply(Node& target, if (key.lerp < 1) { value = values_[key.index - 1].Lerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeTranslation(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeTranslation(value * weight)); } RotationTimelineResolver::RotationTimelineResolver() = default; @@ -108,8 +108,8 @@ void RotationTimelineResolver::Apply(Node& target, if (key.lerp < 1) { value = values_[key.index - 1].Slerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeRotation(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeRotation(value * weight)); } ScaleTimelineResolver::ScaleTimelineResolver() = default; @@ -125,8 +125,8 @@ void ScaleTimelineResolver::Apply(Node& target, SecondsF time, Scalar weight) { if (key.lerp < 1) { value = values_[key.index - 1].Lerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeScale(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeScale(value * weight)); } } // namespace scene diff --git a/impeller/scene/geometry.cc b/impeller/scene/geometry.cc index f39bd7b3b6355..099476d14202d 100644 --- a/impeller/scene/geometry.cc +++ b/impeller/scene/geometry.cc @@ -12,6 +12,8 @@ #include "impeller/geometry/vector.h" #include "impeller/renderer/device_buffer_descriptor.h" #include "impeller/renderer/formats.h" +#include "impeller/renderer/sampler_descriptor.h" +#include "impeller/renderer/sampler_library.h" #include "impeller/renderer/vertex_buffer.h" #include "impeller/renderer/vertex_buffer_builder.h" #include "impeller/scene/importer/scene_flatbuffers.h" @@ -117,6 +119,8 @@ std::shared_ptr Geometry::MakeFromFlatbuffer( return MakeVertexBuffer(std::move(vertex_buffer), is_skinned); } +void Geometry::SetJointsTexture(const std::shared_ptr& texture) {} + //------------------------------------------------------------------------------ /// CuboidGeometry /// @@ -239,10 +243,31 @@ void SkinnedVertexBufferGeometry::BindToCommand( command.BindVertices( GetVertexBuffer(*scene_context.GetContext()->GetResourceAllocator())); + SamplerDescriptor sampler_desc; + sampler_desc.min_filter = MinMagFilter::kNearest; + sampler_desc.mag_filter = MinMagFilter::kNearest; + sampler_desc.mip_filter = MipFilter::kNone; + sampler_desc.width_address_mode = SamplerAddressMode::kRepeat; + sampler_desc.label = "NN Repeat"; + + SkinnedVertexShader::BindJointsTexture( + command, + joints_texture_ ? joints_texture_ : scene_context.GetPlaceholderTexture(), + scene_context.GetContext()->GetSamplerLibrary()->GetSampler( + sampler_desc)); + SkinnedVertexShader::VertInfo info; info.mvp = transform; + info.enable_skinning = joints_texture_ ? 1 : 0; + info.joint_texture_size = + joints_texture_ ? joints_texture_->GetSize().width : 1; SkinnedVertexShader::BindVertInfo(command, buffer.EmplaceUniform(info)); } +// |Geometry| +void SkinnedVertexBufferGeometry::SetJointsTexture( + const std::shared_ptr& texture) { + joints_texture_ = texture; +} } // namespace scene } // namespace impeller diff --git a/impeller/scene/geometry.h b/impeller/scene/geometry.h index b38850080b6e6..269fe46376ad4 100644 --- a/impeller/scene/geometry.h +++ b/impeller/scene/geometry.h @@ -45,6 +45,8 @@ class Geometry { HostBuffer& buffer, const Matrix& transform, Command& command) const = 0; + + virtual void SetJointsTexture(const std::shared_ptr& texture); }; class CuboidGeometry final : public Geometry { @@ -119,8 +121,12 @@ class SkinnedVertexBufferGeometry final : public Geometry { const Matrix& transform, Command& command) const override; + // |Geometry| + void SetJointsTexture(const std::shared_ptr& texture) override; + private: VertexBuffer vertex_buffer_; + std::shared_ptr joints_texture_; FML_DISALLOW_COPY_AND_ASSIGN(SkinnedVertexBufferGeometry); }; diff --git a/impeller/scene/importer/importer_gltf.cc b/impeller/scene/importer/importer_gltf.cc index 2b6ab6322c69c..a6f10b180f298 100644 --- a/impeller/scene/importer/importer_gltf.cc +++ b/impeller/scene/importer/importer_gltf.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -199,22 +200,22 @@ static void ProcessNode(const tinygltf::Model& gltf, /// Matrix transform; - if (in_node.translation.size() == 3) { - transform = transform * Matrix::MakeTranslation( - {static_cast(in_node.translation[0]), - static_cast(in_node.translation[1]), - static_cast(in_node.translation[2])}); + if (in_node.scale.size() == 3) { + transform = + transform * Matrix::MakeScale({static_cast(in_node.scale[0]), + static_cast(in_node.scale[1]), + static_cast(in_node.scale[2])}); } if (in_node.rotation.size() == 4) { transform = transform * Matrix::MakeRotation(Quaternion( in_node.rotation[0], in_node.rotation[1], in_node.rotation[2], in_node.rotation[3])); } - if (in_node.scale.size() == 3) { - transform = - transform * Matrix::MakeScale({static_cast(in_node.scale[0]), - static_cast(in_node.scale[1]), - static_cast(in_node.scale[2])}); + if (in_node.translation.size() == 3) { + transform = transform * Matrix::MakeTranslation( + {static_cast(in_node.translation[0]), + static_cast(in_node.translation[1]), + static_cast(in_node.translation[2])}); } if (in_node.matrix.size() == 16) { if (!transform.IsIdentity()) { @@ -317,11 +318,14 @@ static void ProcessAnimation(const tinygltf::Model& gltf, fb::AnimationT& out_animation) { out_animation.name = in_animation.name; - std::vector> channels; + // std::vector channels; + std::vector translation_channels; + std::vector rotation_channels; + std::vector scale_channels; for (auto& in_channel : in_animation.channels) { - auto out_channel = std::make_unique(); + auto out_channel = fb::ChannelT(); - out_channel->node = in_channel.target_node; + out_channel.node = in_channel.target_node; auto& sampler = in_animation.samplers[in_channel.sampler]; /// Keyframe times. @@ -349,7 +353,7 @@ static void ProcessAnimation(const tinygltf::Model& gltf, const float* time_p = reinterpret_cast( times_buffer.data.data() + times_bufferview.byteOffset + times_accessor.ByteStride(times_bufferview) * time_i); - out_channel->timeline.push_back(*time_p); + out_channel.timeline.push_back(*time_p); } } @@ -387,7 +391,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec3(value_p[0], value_p[1], value_p[2])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + translation_channels.push_back(std::move(out_channel)); } else if (in_channel.target_path == "rotation") { if (values_accessor.type != TINYGLTF_TYPE_VEC4) { std::cerr << "Unexpected type \"" << values_accessor.type @@ -404,7 +409,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec4(value_p[0], value_p[1], value_p[2], value_p[3])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + rotation_channels.push_back(std::move(out_channel)); } else if (in_channel.target_path == "scale") { if (values_accessor.type != TINYGLTF_TYPE_VEC3) { std::cerr << "Unexpected type \"" << values_accessor.type @@ -421,15 +427,22 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec3(value_p[0], value_p[1], value_p[2])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + scale_channels.push_back(std::move(out_channel)); } else { std::cerr << "Unsupported animation channel target path \"" << in_channel.target_path << "\". Skipping." << std::endl; continue; } } + } - channels.push_back(std::move(out_channel)); + std::vector> channels; + for (const auto& channel_list : + {translation_channels, rotation_channels, scale_channels}) { + for (const auto& channel : channel_list) { + channels.push_back(std::make_unique(channel)); + } } out_animation.channels = std::move(channels); } @@ -458,6 +471,9 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) { const tinygltf::Scene& scene = gltf.scenes[gltf.defaultScene]; out_scene.children = scene.nodes; + out_scene.transform = + ToFBMatrixUniquePtr(Matrix::MakeScale(Vector3(1, 1, -1))); + for (size_t texture_i = 0; texture_i < gltf.textures.size(); texture_i++) { auto texture = std::make_unique(); ProcessTexture(gltf, gltf.textures[texture_i], *texture); diff --git a/impeller/scene/importer/importer_unittests.cc b/impeller/scene/importer/importer_unittests.cc index 1c98623068554..de07f76c2d843 100644 --- a/impeller/scene/importer/importer_unittests.cc +++ b/impeller/scene/importer/importer_unittests.cc @@ -74,6 +74,8 @@ TEST(ImporterTest, CanParseSkinnedGLTF) { // The skinned node contains both a skeleton and skinned mesh primitives that // reference bones in the skeleton. auto& skinned_node = scene.nodes[node->children[0]]; + ASSERT_NE(skinned_node->skin, nullptr); + ASSERT_EQ(skinned_node->mesh_primitives.size(), 2u); auto& bottom_triangle = *skinned_node->mesh_primitives[0]; ASSERT_EQ(bottom_triangle.indices->count, 3u); diff --git a/impeller/scene/importer/scene.fbs b/impeller/scene/importer/scene.fbs index bfc949ae7d334..3966e0ed395d9 100644 --- a/impeller/scene/importer/scene.fbs +++ b/impeller/scene/importer/scene.fbs @@ -188,6 +188,7 @@ table Node { table Scene { children: [int]; // Indices into `Scene`->`nodes`. + transform: Matrix; nodes: [Node]; textures: [Texture]; // Textures may be reused across different materials. animations: [Animation]; diff --git a/impeller/scene/importer/vertices_builder.cc b/impeller/scene/importer/vertices_builder.cc index 14f56dc214d54..78bde249f6b53 100644 --- a/impeller/scene/importer/vertices_builder.cc +++ b/impeller/scene/importer/vertices_builder.cc @@ -65,19 +65,6 @@ static void PassthroughAttributeWriter( } } -/// @brief A ComponentWriter which converts a Vector3 position from -/// right-handed GLTF space to left-handed Impeller space. -static void PositionAttributeWriter( - Scalar* destination, - const void* source, - const VerticesBuilder::ComponentProperties& component, - const VerticesBuilder::AttributeProperties& attribute) { - FML_DCHECK(attribute.component_count == 3); - *(destination + 0) = component.convert_proc(source, 0, true); - *(destination + 1) = component.convert_proc(source, 1, true); - *(destination + 2) = -component.convert_proc(source, 2, true); -} - /// @brief A ComponentWriter which converts four vertex indices to scalars. static void JointsAttributeWriter( Scalar* destination, @@ -96,7 +83,7 @@ std::map {.offset_bytes = offsetof(UnskinnedVerticesBuilder::Vertex, position), .size_bytes = sizeof(UnskinnedVerticesBuilder::Vertex::position), .component_count = 3, - .write_proc = PositionAttributeWriter}}, + .write_proc = PassthroughAttributeWriter}}, {VerticesBuilder::AttributeType::kNormal, {.offset_bytes = offsetof(UnskinnedVerticesBuilder::Vertex, normal), .size_bytes = sizeof(UnskinnedVerticesBuilder::Vertex::normal), diff --git a/impeller/scene/mesh.cc b/impeller/scene/mesh.cc index ff9558126285c..bc4b02da00a80 100644 --- a/impeller/scene/mesh.cc +++ b/impeller/scene/mesh.cc @@ -5,9 +5,11 @@ #include "impeller/scene/mesh.h" #include +#include #include "impeller/base/validation.h" #include "impeller/scene/material.h" +#include "impeller/scene/pipeline_key.h" #include "impeller/scene/scene_encoder.h" namespace impeller { @@ -31,8 +33,11 @@ std::vector& Mesh::GetPrimitives() { return primitives_; } -bool Mesh::Render(SceneEncoder& encoder, const Matrix& transform) const { +bool Mesh::Render(SceneEncoder& encoder, + const Matrix& transform, + const std::shared_ptr& joints) const { for (const auto& mesh : primitives_) { + mesh.geometry->SetJointsTexture(joints); SceneCommand command = { .label = "Mesh Primitive", .transform = transform, diff --git a/impeller/scene/mesh.h b/impeller/scene/mesh.h index 5684b4fa5d8dd..dc33caf27b1c3 100644 --- a/impeller/scene/mesh.h +++ b/impeller/scene/mesh.h @@ -15,6 +15,8 @@ namespace impeller { namespace scene { +class Skin; + class Mesh final { public: struct Primitive { @@ -31,7 +33,9 @@ class Mesh final { void AddPrimitive(Primitive mesh_); std::vector& GetPrimitives(); - bool Render(SceneEncoder& encoder, const Matrix& transform) const; + bool Render(SceneEncoder& encoder, + const Matrix& transform, + const std::shared_ptr& joints) const; private: std::vector primitives_; diff --git a/impeller/scene/node.cc b/impeller/scene/node.cc index 72a551af71278..dcc9b2c213278 100644 --- a/impeller/scene/node.cc +++ b/impeller/scene/node.cc @@ -126,6 +126,8 @@ std::shared_ptr Node::MakeFromFlatbuffer(const fb::Scene& scene, } auto result = std::make_shared(); + result->SetLocalTransform(importer::ToMatrix(*scene.transform())); + if (!scene.nodes() || !scene.children()) { return result; // The scene is empty. } @@ -190,17 +192,21 @@ void Node::UnpackFromFlatbuffer( /// Child nodes. - if (!source_node.children()) { - return; + if (source_node.children()) { + // Wire up graph connections. + for (int child : *source_node.children()) { + if (child < 0 || static_cast(child) >= scene_nodes.size()) { + VALIDATION_LOG << "Node child index out of range."; + continue; + } + AddChild(scene_nodes[child]); + } } - // Wire up graph connections. - for (int child : *source_node.children()) { - if (child < 0 || static_cast(child) >= scene_nodes.size()) { - VALIDATION_LOG << "Node child index out of range."; - continue; - } - AddChild(scene_nodes[child]); + /// Skin. + + if (source_node.skin()) { + skin_ = Skin::MakeFromFlatbuffer(*source_node.skin(), scene_nodes); } } @@ -220,6 +226,10 @@ void Node::SetName(const std::string& new_name) { name_ = new_name; } +Node* Node::GetParent() const { + return parent_; +} + std::shared_ptr Node::FindChildByName( const std::string& name, bool exclude_animation_players) const { @@ -301,16 +311,27 @@ Mesh& Node::GetMesh() { return mesh_; } -bool Node::Render(SceneEncoder& encoder, const Matrix& parent_transform) const { +void Node::SetIsJoint(bool is_joint) { + is_joint_ = is_joint; +} + +bool Node::IsJoint() const { + return is_joint_; +} + +bool Node::Render(SceneEncoder& encoder, + Allocator& allocator, + const Matrix& parent_transform) const { if (animation_player_.has_value()) { animation_player_->Update(); } Matrix transform = parent_transform * local_transform_; - mesh_.Render(encoder, transform); + mesh_.Render(encoder, transform, + skin_ ? skin_->GetJointsTexture(allocator) : nullptr); for (auto& child : children_) { - if (!child->Render(encoder, transform)) { + if (!child->Render(encoder, allocator, transform)) { return false; } } diff --git a/impeller/scene/node.h b/impeller/scene/node.h index e573f9991b236..8b79e5f90fd81 100644 --- a/impeller/scene/node.h +++ b/impeller/scene/node.h @@ -18,6 +18,7 @@ #include "impeller/scene/camera.h" #include "impeller/scene/mesh.h" #include "impeller/scene/scene_encoder.h" +#include "impeller/scene/skin.h" namespace impeller { namespace scene { @@ -36,6 +37,8 @@ class Node final { const std::string& GetName() const; void SetName(const std::string& new_name); + Node* GetParent() const; + std::shared_ptr FindChildByName( const std::string& name, bool exclude_animation_players = false) const; @@ -55,7 +58,12 @@ class Node final { void SetMesh(Mesh mesh); Mesh& GetMesh(); - bool Render(SceneEncoder& encoder, const Matrix& parent_transform) const; + void SetIsJoint(bool is_joint); + bool IsJoint() const; + + bool Render(SceneEncoder& encoder, + Allocator& allocator, + const Matrix& parent_transform) const; private: void UnpackFromFlatbuffer( @@ -68,6 +76,7 @@ class Node final { std::string name_; bool is_root_ = false; + bool is_joint_ = false; Node* parent_ = nullptr; std::vector> children_; Mesh mesh_; @@ -76,6 +85,8 @@ class Node final { std::vector> animations_; mutable std::optional animation_player_; + std::unique_ptr skin_; + FML_DISALLOW_COPY_AND_ASSIGN(Node); friend Scene; diff --git a/impeller/scene/scene.cc b/impeller/scene/scene.cc index cf88faa10cfc4..d1c8cae5002b2 100644 --- a/impeller/scene/scene.cc +++ b/impeller/scene/scene.cc @@ -34,7 +34,9 @@ bool Scene::Render(const RenderTarget& render_target, const Matrix& camera_transform) const { // Collect the render commands from the scene. SceneEncoder encoder; - if (!root_.Render(encoder, Matrix())) { + if (!root_.Render(encoder, + *scene_context_->GetContext()->GetResourceAllocator(), + Matrix())) { FML_LOG(ERROR) << "Failed to render frame."; return false; } diff --git a/impeller/scene/scene_context.cc b/impeller/scene/scene_context.cc index 3f1ab5ba6cd85..284d6da6b3ac1 100644 --- a/impeller/scene/scene_context.cc +++ b/impeller/scene/scene_context.cc @@ -51,14 +51,15 @@ SceneContext::SceneContext(std::shared_ptr context) placeholder_texture_ = context_->GetResourceAllocator()->CreateTexture(texture_descriptor); + placeholder_texture_->SetLabel("Placeholder Texture"); if (!placeholder_texture_) { - FML_DLOG(ERROR) << "Could not create placeholder texture."; + FML_LOG(ERROR) << "Could not create placeholder texture."; return; } uint8_t pixel[] = {0xFF, 0xFF, 0xFF, 0xFF}; - if (!placeholder_texture_->SetContents(pixel, 4, 0)) { - FML_DLOG(ERROR) << "Could not set contents of placeholder texture."; + if (!placeholder_texture_->SetContents(pixel, 4)) { + FML_LOG(ERROR) << "Could not set contents of placeholder texture."; return; } } diff --git a/impeller/scene/scene_unittests.cc b/impeller/scene/scene_unittests.cc index 5ce513ab165e2..f2a048f8a7f4f 100644 --- a/impeller/scene/scene_unittests.cc +++ b/impeller/scene/scene_unittests.cc @@ -135,6 +135,31 @@ TEST_P(SceneTest, TwoTriangles) { scene.GetRoot().AddChild(std::move(gltf_scene)); Renderer::RenderCallback callback = [&](RenderTarget& render_target) { + ImGui::Begin("Controls", nullptr, ImGuiWindowFlags_AlwaysAutoResize); + { + static Scalar playback_time_scale = 1; + static Scalar weight = 1; + static bool loop = true; + + ImGui::SliderFloat("Playback time scale", &playback_time_scale, -5, 5); + ImGui::SliderFloat("Weight", &weight, -2, 2); + ImGui::Checkbox("Loop", &loop); + if (ImGui::Button("Play")) { + metronome_clip.Play(); + } + if (ImGui::Button("Pause")) { + metronome_clip.Pause(); + } + if (ImGui::Button("Stop")) { + metronome_clip.Stop(); + } + + metronome_clip.SetPlaybackTimeScale(playback_time_scale); + metronome_clip.SetWeight(weight); + metronome_clip.SetLoop(loop); + } + + ImGui::End(); Node& node = *scene.GetRoot().GetChildren()[0]; node.SetLocalTransform(node.GetLocalTransform() * Matrix::MakeRotation(0.02, {0, 1, 0, 0})); diff --git a/impeller/scene/shaders/skinned.vert b/impeller/scene/shaders/skinned.vert index 887ce8f1f3778..884b204d796ba 100644 --- a/impeller/scene/shaders/skinned.vert +++ b/impeller/scene/shaders/skinned.vert @@ -4,9 +4,13 @@ uniform VertInfo { mat4 mvp; + float enable_skinning; + float joint_texture_size; } vert_info; +uniform sampler2D joints_texture; + // This attribute layout is expected to be identical to `SkinnedVertex` within // `impeller/scene/importer/scene.fbs`. in vec3 position; @@ -14,7 +18,6 @@ in vec3 normal; in vec4 tangent; in vec2 texture_coords; in vec4 color; -// TODO(bdero): Use the joint indices to sample bone matrices from a texture. in vec4 joints; in vec4 weights; @@ -23,18 +26,50 @@ out mat3 v_tangent_space; out vec2 v_texture_coords; out vec4 v_color; +const int kMatrixTexelStride = 4; + +mat4 GetJoint(float joint_index) { + // The size of one texel in UV space. The joint texture should always be + // square, so the answer is the same in both dimensions. + float texel_size_uv = 1 / vert_info.joint_texture_size; + + // Each joint matrix takes up 4 pixels (16 floats), so we jump 4 pixels per + // joint matrix. + float matrix_start = joint_index * kMatrixTexelStride; + + // The texture space coordinates at the start of the matrix. + float x = mod(matrix_start, vert_info.joint_texture_size); + float y = floor(matrix_start / vert_info.joint_texture_size); + + // Nearest sample the middle of each the texel by adding `0.5 * texel_size_uv` + // to both dimensions. + y = (y + 0.5) * texel_size_uv; + mat4 joint = + mat4(texture(joints_texture, vec2((x + 0.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 1.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 2.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 3.5) * texel_size_uv, y))); + + return joint; +} + void main() { - // The following two lines are temporary placeholders to prevent the vertex - // attributes from being removed from the shader. - v_color = joints; - v_color = weights; + mat4 skin_matrix; + if (vert_info.enable_skinning == 1) { + skin_matrix = + GetJoint(joints.x) * weights.x + GetJoint(joints.y) * weights.y + + GetJoint(joints.z) * weights.z + GetJoint(joints.w) * weights.w; + } else { + skin_matrix = mat4(1); // Identity matrix. + } - gl_Position = vert_info.mvp * vec4(position, 1.0); + gl_Position = vert_info.mvp * skin_matrix * vec4(position, 1.0); v_position = gl_Position.xyz; - vec3 lh_tangent = tangent.xyz * tangent.w; - v_tangent_space = - mat3(vert_info.mvp) * mat3(lh_tangent, cross(normal, lh_tangent), normal); + vec3 lh_tangent = (skin_matrix * vec4(tangent.xyz * tangent.w, 0.0)).xyz; + vec3 out_normal = (skin_matrix * vec4(normal, 0.0)).xyz; + v_tangent_space = mat3(vert_info.mvp) * + mat3(lh_tangent, cross(out_normal, lh_tangent), out_normal); v_texture_coords = texture_coords; v_color = color; } diff --git a/impeller/scene/skin.cc b/impeller/scene/skin.cc new file mode 100644 index 0000000000000..4a4c614195289 --- /dev/null +++ b/impeller/scene/skin.cc @@ -0,0 +1,121 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/scene/skin.h" + +#include +#include +#include +#include + +#include "flutter/fml/logging.h" +#include "impeller/renderer/allocator.h" +#include "impeller/scene/importer/conversions.h" + +namespace impeller { +namespace scene { + +std::unique_ptr Skin::MakeFromFlatbuffer( + const fb::Skin& skin, + const std::vector>& scene_nodes) { + if (!skin.joints() || !skin.inverse_bind_matrices() || + skin.joints()->size() != skin.inverse_bind_matrices()->size()) { + VALIDATION_LOG << "Skin data is missing joints or bind matrices."; + return nullptr; + } + + Skin result; + + result.joints_.reserve(skin.joints()->size()); + for (auto joint : *skin.joints()) { + if (joint < 0 || static_cast(joint) > scene_nodes.size()) { + VALIDATION_LOG << "Skin joint index out of range."; + result.joints_.push_back(nullptr); + continue; + } + if (scene_nodes[joint]) { + scene_nodes[joint]->SetIsJoint(true); + } + result.joints_.push_back(scene_nodes[joint]); + } + + result.inverse_bind_matrices_.reserve(skin.inverse_bind_matrices()->size()); + for (auto matrix : *skin.inverse_bind_matrices()) { + if (!matrix) { + result.inverse_bind_matrices_.push_back(Matrix()); + continue; + } + result.inverse_bind_matrices_.push_back(importer::ToMatrix(*matrix)); + } + + return std::make_unique(std::move(result)); +} + +Skin::Skin() = default; + +Skin::~Skin() = default; + +Skin::Skin(Skin&&) = default; + +Skin& Skin::operator=(Skin&&) = default; + +std::shared_ptr Skin::GetJointsTexture(Allocator& allocator) { + // Each joint has a matrix. 1 matrix = 16 floats. 1 pixel = 4 floats. + // Therefore, each joint needs 4 pixels. + auto required_pixels = joints_.size() * 4; + auto dimension_size = std::max( + 2u, + Allocation::NextPowerOfTwoSize(std::ceil(std::sqrt(required_pixels)))); + + impeller::TextureDescriptor texture_descriptor; + texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + texture_descriptor.format = PixelFormat::kR32G32B32A32Float; + texture_descriptor.size = {dimension_size, dimension_size}; + texture_descriptor.mip_count = 1u; + + auto result = allocator.CreateTexture(texture_descriptor); + result->SetLabel("Joints Texture"); + if (!result) { + FML_LOG(ERROR) << "Could not create joint texture."; + return nullptr; + } + + std::vector joints; + joints.resize(result->GetSize().Area() / 4, Matrix()); + FML_DCHECK(joints.size() >= joints_.size()); + for (size_t joint_i = 0; joint_i < joints_.size(); joint_i++) { + const Node* joint = joints_[joint_i].get(); + if (!joint) { + // When a joint is missing, just let it remain as an identity matrix. + continue; + } + + // Compute a model space matrix for the joint by walking up the bones to the + // skeleton root. + while (joint && joint->IsJoint()) { + joints[joint_i] = joint->GetLocalTransform() * joints[joint_i]; + joint = joint->GetParent(); + } + + // Get the joint transform relative to the default pose of the bone by + // incorporating the joint's inverse bind matrix. The inverse bind matrix + // transforms from model space to the default pose space of the joint. The + // result is a model space matrix that only captures the difference between + // the joint's default pose and the joint's current pose in the scene. This + // is necessary because the skinned model's vertex positions (which _define_ + // the default pose) are all in model space. + joints[joint_i] = joints[joint_i] * inverse_bind_matrices_[joint_i]; + } + + if (!result->SetContents(reinterpret_cast(joints.data()), + joints.size() * sizeof(Matrix))) { + FML_LOG(ERROR) << "Could not set contents of joint texture."; + return nullptr; + } + + return result; +} + +} // namespace scene +} // namespace impeller diff --git a/impeller/scene/skin.h b/impeller/scene/skin.h new file mode 100644 index 0000000000000..016b527e771c0 --- /dev/null +++ b/impeller/scene/skin.h @@ -0,0 +1,42 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include + +#include "flutter/fml/macros.h" + +#include "impeller/renderer/allocator.h" +#include "impeller/renderer/texture.h" +#include "impeller/scene/importer/scene_flatbuffers.h" +#include "impeller/scene/node.h" + +namespace impeller { +namespace scene { + +class Skin final { + public: + static std::unique_ptr MakeFromFlatbuffer( + const fb::Skin& skin, + const std::vector>& scene_nodes); + ~Skin(); + + Skin(Skin&&); + Skin& operator=(Skin&&); + + std::shared_ptr GetJointsTexture(Allocator& allocator); + + private: + Skin(); + + std::vector> joints_; + std::vector inverse_bind_matrices_; + + FML_DISALLOW_COPY_AND_ASSIGN(Skin); +}; + +} // namespace scene +} // namespace impeller