Fix incomplete codepoints in IO#each_codepoint.

See https://bugs.ruby-lang.org/issues/11444 Does not appear to fix all of MRI's TestIO_M17N#test_each_codepoint_need_more. Passes binary case, but I'm not sure if that's valuable or not.
jruby · Mar 3, 2016 · 866c248 · 866c248
1 parent 876445b
commit 866c248
Showing 2 changed files with 23 additions and 0 deletions.
diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java
@@ -3189,6 +3189,24 @@ private IRubyObject eachCodePointCommon(ThreadContext context, Block block, Stri
                     block.yield(context, runtime.newFixnum(c & 0xFFFFFFFF));
                 } else if (StringSupport.MBCLEN_INVALID_P(r)) {
                     throw runtime.newArgumentError("invalid byte sequence in " + enc.toString());
+                } else if (StringSupport.MBCLEN_NEEDMORE_P(r)) {
+                    byte[] cbuf = new byte[8];
+                    int p = 0;
+                    int more = StringSupport.MBCLEN_NEEDMORE_LEN(r);
+                    if (more > cbuf.length) throw runtime.newArgumentError("invalid byte sequence in " + enc.toString());
+                    more += n = fptr.rbuf.len;
+                    if (more > cbuf.length) throw runtime.newArgumentError("invalid byte sequence in " + enc.toString());
+                    while ((n = (int)fptr.readBufferedData(cbuf, p, more)) > 0) {
+                        p += n;
+                        if ((more -= n) <= 0) break;
+
+                        if (fptr.fillbuf(context) < 0) throw runtime.newArgumentError("invalid byte sequence in " + enc.toString());
+                        if ((n = fptr.rbuf.len) > more) n = more;
+                    }
+                    r = enc.length(cbuf, 0, p);
+                    if (!StringSupport.MBCLEN_CHARFOUND_P(r)) throw runtime.newArgumentError("invalid byte sequence in " + enc.toString());
+                    c = enc.mbcToCode(cbuf, 0, p);
+                    block.yield(context, runtime.newFixnum(c));
                 } else {
                     continue;
                 }

diff --git a/core/src/main/java/org/jruby/util/StringSupport.java b/core/src/main/java/org/jruby/util/StringSupport.java
@@ -109,6 +109,11 @@ public static int MBCLEN_NEEDMORE(int n) {
         return -1 - n;
     }
 
+    // MBCLEN_NEEDMORE_LEN, ONIGENC_MBCLEN_NEEDMORE_LEN
+    public static int MBCLEN_NEEDMORE_LEN(int r) {
+        return -1 - r;
+    }
+
     // MBCLEN_INVALID_P, ONIGENC_MBCLEN_INVALID_P
     public static boolean MBCLEN_INVALID_P(int r) {
         return r == -1;