Avoid fast-path IO writes when IO has ext enc

This works around a bug in JRuby's IOOutputStream logic whereby an
IO with an external encoding will always fail to write incoming
bytes. We use base logic to detect if the target object is a "real
IO" and if so and it has an external encoding, we drop the realIO
and. This causes the rest of IOOutputStream to avoid the fast path
and always use dyncall logic with RubyString wrappers.

This works around the issue in jruby/jruby#8682.

This should be temporary, since it will definitely degrade direct
writes to such IO objects, but a longer-term fix for the encoding
issues spelled out in jruby/jruby#8682 will need to come first,
or else json will have to be modified to not use IOOutputStream at
all.

Author: headius

java/src/json/ext/Generator.java

@@ -5,6 +5,7 @@
  */
 package json.ext;
 
+import org.jcodings.Encoding;
 import org.jcodings.specific.UTF8Encoding;
 import org.jruby.Ruby;
 import org.jruby.RubyArray;
@@ -15,6 +16,7 @@
 import org.jruby.RubyFixnum;
 import org.jruby.RubyFloat;
 import org.jruby.RubyHash;
+import org.jruby.RubyIO;
 import org.jruby.RubyString;
 import org.jruby.RubySymbol;
 import org.jruby.runtime.Helpers;
@@ -81,11 +83,41 @@ static <T extends IRubyObject> RubyString generateJson(ThreadContext context, T
             return handler.generateNew(context, session, object);
         }
 
-        BufferedOutputStream buffer = new BufferedOutputStream(new IOOutputStream(io), IO_BUFFER_SIZE);
+        BufferedOutputStream buffer =
+                new BufferedOutputStream(
+                        new PatchedIOOutputStream(io, UTF8Encoding.INSTANCE),
+                        IO_BUFFER_SIZE);
         handler.generateToBuffer(context, session, object, buffer);
         return io;
     }
 
+    /**
+     * A version of IOOutputStream hacked to avoid fast-path RubyIO calls when the target IO has an external encoding.
+     *
+     * All calls to the underlying IO will be done dynamically and all incoming bytes wrapped in RubyString instances.
+     * This avoids bugs in the fast-path logic in JRuby 9.4.12.0 and earlier that fails to properly handle writing bytes
+     * when the source and target destination are the same.
+     *
+     * See https://github.com/jruby/jruby/issues/8682
+     */
+    private static class PatchedIOOutputStream extends IOOutputStream {
+        public PatchedIOOutputStream(IRubyObject io, Encoding encoding) {
+            super(io, encoding);
+        }
+
+        @Override
+        public RubyIO getRealIO(IRubyObject io) {
+            RubyIO realIO = super.getRealIO(io);
+
+            // if the real IO has an external encoding, don't use fast path
+            if (realIO == null || realIO.getEnc() != null) {
+                return null;
+            }
+
+            return realIO;
+        }
+    }
+
     /**
      * Returns the best serialization handler for the given object.
      */