diff --git a/base/summarysize.jl b/base/summarysize.jl
index 2505824768099..d6edad2ad3150 100644
--- a/base/summarysize.jl
+++ b/base/summarysize.jl
@@ -56,9 +56,13 @@ function summarysize(obj;
             end
         else
             nf = nfields(x)
-            ft = typeof(x).types
-            if !isbitstype(ft[i]) && isdefined(x, i)
-                val = getfield(x, i)
+            dt = typeof(x)
+            dtfd = Base.DataTypeFieldDesc(dt)
+            if isdefined(x, i)
+                f = getfield(x, i)
+                if dtfd[i].isptr || !Base.datatype_pointerfree(typeof(f))
+                    val = f
+                end
             end
         end
         if nf > i
diff --git a/test/misc.jl b/test/misc.jl
index 6c2f0f89135fd..020c1184788c5 100644
--- a/test/misc.jl
+++ b/test/misc.jl
@@ -573,6 +573,17 @@ end
 # issue #44780
 @test summarysize(BigInt(2)^1000) > summarysize(BigInt(2))
 
+# issue #53061
+mutable struct S53061
+    x::Union{Float64, Tuple{Float64, Float64}}
+    y::Union{Float64, Tuple{Float64, Float64}}
+end
+let s = S53061[S53061(rand(), (rand(),rand())) for _ in 1:10^5]
+    sz1 = summarysize(s)
+    sz2 = summarysize(s)
+    @test sz1 == sz2
+end
+
 ## test conversion from UTF-8 to UTF-16 (for Windows APIs)
 
 # empty arrays