align C ported memsearch code - need to handle array[length] properly

... otherwise code doing utf8 str scan might fail with array-out-of-index resolves GH-2036
jruby · Feb 8, 2018 · bc39a0a · bc39a0a
1 parent edd18dd
commit bc39a0a
Showing 3 changed files with 26 additions and 7 deletions.
diff --git a/core/src/main/java/org/jruby/RubyString.java b/core/src/main/java/org/jruby/RubyString.java
@@ -2728,7 +2728,6 @@ private IRubyObject indexCommon19(Ruby runtime, ThreadContext context, IRubyObje
 
     // MRI: rb_strseq_index
     private int strseqIndex(final RubyString sub, int offset, boolean inBytes) {
-        byte[] sBytes = value.unsafeBytes();
         int s, sptr, e;
         int pos, len, slen;
         boolean single_byte = singleByteOptimizable();
@@ -2745,6 +2744,7 @@ private int strseqIndex(final RubyString sub, int offset, boolean inBytes) {
         }
         if (len - offset < slen) return -1;
 
+        byte[] sBytes = value.unsafeBytes();
         s = value.begin();
         e = s + value.realSize();
         if (offset != 0) {
@@ -3811,7 +3811,7 @@ public IRubyObject scan19(ThreadContext context, IRubyObject pat, Block block) {
         mustnotBroken(context);
         if (!block.isGiven()) {
             RubyArray ary = null;
-            while (!(result = scanOnce(context, str, pat, startp)).isNil()) {
+            while ((result = scanOnce(context, str, pat, startp)) != context.nil) {
                 last = prev;
                 prev = startp[0];
                 if (ary == null) ary = context.runtime.newArray(4);
@@ -3824,7 +3824,7 @@ public IRubyObject scan19(ThreadContext context, IRubyObject pat, Block block) {
         final byte[] pBytes = value.unsafeBytes();
         final int len = value.realSize();
 
-        while (!(result = scanOnce(context, str, pat, startp)).isNil()) {
+        while ((result = scanOnce(context, str, pat, startp)) != context.nil) {
             last = prev;
             prev = startp[0];
             block.yieldSpecific(context, result);

diff --git a/core/src/main/java/org/jruby/util/StringSupport.java b/core/src/main/java/org/jruby/util/StringSupport.java
@@ -2297,9 +2297,15 @@ private static int rb_memsearch_qs(byte[] xsBytes, int xs, int m, byte[] ysBytes
         return -1;
     }
 
-    private static int rb_memsearch_qs_utf8_hash(byte[] xBytes, int x) {
-        int mix = 8353;
-        int h = xBytes[x] & 0xFF;
+    private static int rb_memsearch_qs_utf8_hash(byte[] xBytes, final int x) {
+        final int mix = 8353;
+        int h;
+        if (x != xBytes.length) {
+            h = xBytes[x] & 0xFF;
+        }
+        else {
+            h = '\0'; // (C) ary end - due y+m at rb_memsearch_qs_utf8
+        }
         if (h < 0xC0) {
             return h + 256;
         }
@@ -2337,7 +2343,7 @@ private static int rb_memsearch_qs_utf8(byte[] xsBytes, int xs, int m, byte[] ys
         for (; x < xe; ++x) {
             qstable[rb_memsearch_qs_utf8_hash(xsBytes, x)] = xe - x;
         }
-        /* Searching */
+        /* Searching */ // due y+m <= ... (y+m) might == ary.length
         for (; y + m <= ys + n; y += qstable[rb_memsearch_qs_utf8_hash(ysBytes, y+m)]) {
             if (xsBytes[xs] == ysBytes[y] && ByteList.memcmp(xsBytes, xs, ysBytes, y, m) == 0)
                 return y - ys;

diff --git a/test/jruby/test_string.rb b/test/jruby/test_string.rb
@@ -135,4 +135,17 @@ def try(obj, *a, &b) # ~ AS 4.2
     end
   end
 
+  public
+
+  def test_scan_error
+    string = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"
+    assert_equal [], 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz'.scan('d....r........')
+
+    ('a'..'z').to_a.each do |c1|
+      ('a'..'z').to_a.each do |c2|
+        string.downcase.scan("#{c1}....#{c2}........") # does not blow with ArrayIndexOutOfBoundsException
+      end
+    end
+  end
+
 end