jruby · Jun 8, 2015 · Jun 8, 2015 · Jun 8, 2015 · Jun 8, 2015
diff --git a/core/src/main/java/org/jruby/RubyMatchData.java b/core/src/main/java/org/jruby/RubyMatchData.java
@@ -50,6 +50,7 @@
 import org.jruby.runtime.Visibility;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
+import org.jruby.util.ByteListHolder;
 import org.jruby.util.StringSupport;
 
 /**
@@ -398,7 +399,7 @@ private int nameToBackrefNumber(RubyString str) {
         return nameToBackrefNumber(getRuntime(), pattern, regs, str);
     }
 
-    private static int nameToBackrefNumber(Ruby runtime, Regex pattern, Region regs, RubyString str) {
+    private static int nameToBackrefNumber(Ruby runtime, Regex pattern, Region regs, ByteListHolder str) {
         if (pattern == null) {
             throw runtime.newIndexError("undefined group name reference: " + str);
         }

diff --git a/spec/truffle/tags/core/string/element_set_tags.txt b/spec/truffle/tags/core/string/element_set_tags.txt
diff --git a/spec/truffle/tags/core/string/index_tags.txt b/spec/truffle/tags/core/string/index_tags.txt
diff --git a/spec/truffle/tags/core/string/match_tags.txt b/spec/truffle/tags/core/string/match_tags.txt
diff --git a/truffle/src/main/java/org/jruby/truffle/nodes/core/MatchDataNodes.java b/truffle/src/main/java/org/jruby/truffle/nodes/core/MatchDataNodes.java
@@ -21,6 +21,8 @@
 import org.jruby.truffle.nodes.coerce.ToIntNode;
 import org.jruby.truffle.nodes.coerce.ToIntNodeGen;
 import org.jruby.truffle.nodes.core.array.ArrayNodes;
+import org.jruby.truffle.nodes.dispatch.CallDispatchHeadNode;
+import org.jruby.truffle.nodes.dispatch.DispatchHeadNodeFactory;
 import org.jruby.truffle.runtime.NotProvided;
 import org.jruby.truffle.runtime.RubyContext;
 import org.jruby.truffle.runtime.control.RaiseException;
@@ -190,6 +192,38 @@ public Object end(RubyMatchData matchData, int index) {
         }
     }
 
+    @RubiniusOnly
+    @CoreMethod(names = "full")
+    public abstract static class FullNode extends CoreMethodArrayArgumentsNode {
+
+        @Child private CallDispatchHeadNode newTupleNode;
+
+        public FullNode(RubyContext context, SourceSection sourceSection) {
+            super(context, sourceSection);
+        }
+
+        @Specialization
+        public Object full(VirtualFrame frame, RubyMatchData matchData) {
+            if (matchData.getFullTuple() != null) {
+                return matchData.getFullTuple();
+            }
+
+            if (newTupleNode == null) {
+                CompilerDirectives.transferToInterpreter();
+                newTupleNode = insert(DispatchHeadNodeFactory.createMethodCall(getContext()));
+            }
+
+            final Object fullTuple = newTupleNode.call(frame,
+                    getContext().getCoreLibrary().getTupleClass(),
+                    "create",
+                    null, matchData.getFullBegin(), matchData.getFullEnd());
+
+            matchData.setFullTuple(fullTuple);
+
+            return fullTuple;
+        }
+    }
+
     @CoreMethod(names = {"length", "size"})
     public abstract static class LengthNode extends CoreMethodArrayArgumentsNode {
 

diff --git a/truffle/src/main/java/org/jruby/truffle/nodes/core/StringNodes.java b/truffle/src/main/java/org/jruby/truffle/nodes/core/StringNodes.java
@@ -40,8 +40,6 @@
 import org.jcodings.specific.ASCIIEncoding;
 import org.jcodings.specific.USASCIIEncoding;
 import org.jcodings.specific.UTF8Encoding;
-import org.joni.Matcher;
-import org.joni.Option;
 import org.jruby.Ruby;
 import org.jruby.truffle.nodes.RubyGuards;
 import org.jruby.truffle.nodes.RubyNode;

diff --git a/truffle/src/main/java/org/jruby/truffle/nodes/rubinius/RegexpPrimitiveNodes.java b/truffle/src/main/java/org/jruby/truffle/nodes/rubinius/RegexpPrimitiveNodes.java
@@ -14,6 +14,7 @@
 import com.oracle.truffle.api.dsl.ImportStatic;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.source.SourceSection;
+import org.jcodings.Encoding;
 import org.joni.Matcher;
 import org.joni.Regex;
 import org.jruby.truffle.nodes.core.RegexpGuards;
@@ -25,6 +26,7 @@
 import org.jruby.truffle.runtime.core.RubyRegexp;
 import org.jruby.truffle.runtime.core.RubyString;
 import org.jruby.util.ByteList;
+import org.jruby.util.RegexpSupport;
 import org.jruby.util.StringSupport;
 
 /**
@@ -138,7 +140,10 @@ public Object searchRegionInvalidEncoding(RubyRegexp regexp, RubyString string,
         @Specialization(guards = { "isInitialized(regexp)", "isValidEncoding(string)" })
         public Object searchRegion(RubyRegexp regexp, RubyString string, int start, int end, boolean forward) {
             final ByteList bl = regexp.getSource();
-            final Regex r = new Regex(bl.getUnsafeBytes(), bl.getBegin(), bl.getBegin() + bl.getRealSize(), regexp.getRegex().getOptions(), regexp.checkEncoding(StringNodes.getCodeRangeable(string), true));
+            final Encoding enc = regexp.checkEncoding(StringNodes.getCodeRangeable(string), true);
+            final ByteList preprocessed = RegexpSupport.preprocess(getContext().getRuntime(), bl, enc, new Encoding[]{null}, RegexpSupport.ErrorMode.RAISE);
+
+            final Regex r = new Regex(preprocessed.getUnsafeBytes(), preprocessed.getBegin(), preprocessed.getBegin() + preprocessed.getRealSize(), regexp.getRegex().getOptions(), regexp.checkEncoding(StringNodes.getCodeRangeable(string), true));
             final Matcher matcher = r.matcher(StringNodes.getByteList(string).bytes());
 
             return regexp.matchCommon(string, false, false, matcher, start, end);

diff --git a/truffle/src/main/java/org/jruby/truffle/runtime/core/RubyMatchData.java b/truffle/src/main/java/org/jruby/truffle/runtime/core/RubyMatchData.java
@@ -6,13 +6,41 @@
  * Eclipse Public License version 1.0
  * GNU General Public License version 2
  * GNU Lesser General Public License version 2.1
+ *
+ * Some of the code in this class is transposed from org.jruby.RubyMatchData,
+ * licensed under the same EPL1.0/GPL 2.0/LGPL 2.1 used throughout.
+ *
+ * Copyright (C) 2001 Alan Moore <alan_moore@gmx.net>
+ * Copyright (C) 2001-2004 Jan Arne Petersen <jpetersen@uni-bonn.de>
+ * Copyright (C) 2002 Benoit Cerrina <b.cerrina@wanadoo.fr>
+ * Copyright (C) 2002-2004 Anders Bengtsson <ndrsbngtssn@yahoo.se>
+ * Copyright (C) 2004 Thomas E Enebo <enebo@acm.org>
+ * Copyright (C) 2004 Charles O Nutter <headius@headius.com>
+ * Copyright (C) 2004 Stefan Matthias Aust <sma@3plus4.de>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either of the GNU General Public License Version 2 or later (the "GPL"),
+ * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the EPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the EPL, the GPL or the LGPL.
  */
 package org.jruby.truffle.runtime.core;
 
+import org.jcodings.Encoding;
 import org.joni.Region;
+import org.jruby.truffle.nodes.core.StringNodes;
 import org.jruby.truffle.runtime.array.ArrayUtils;
+import org.jruby.truffle.runtime.control.RaiseException;
 import org.jruby.truffle.runtime.subsystems.ObjectSpaceManager;
 import org.jruby.util.ByteList;
+import org.jruby.util.CodeRangeable;
+import org.jruby.util.StringSupport;
 
 import java.util.Arrays;
 
@@ -28,8 +56,12 @@ public class RubyMatchData extends RubyBasicObject {
     private final RubyBasicObject pre;
     private final RubyBasicObject post;
     private final RubyBasicObject global;
+    boolean charOffsetUpdated;
+    Region charOffsets;
+    private final int begin, end;
+    private Object fullTuple;
 
-    public RubyMatchData(RubyClass rubyClass, RubyBasicObject source, RubyRegexp regexp, Region region, Object[] values, RubyBasicObject pre, RubyBasicObject post, RubyBasicObject global) {
+    public RubyMatchData(RubyClass rubyClass, RubyBasicObject source, RubyRegexp regexp, Region region, Object[] values, RubyBasicObject pre, RubyBasicObject post, RubyBasicObject global, int begin, int end) {
         super(rubyClass);
         this.source = source;
         this.regexp = regexp;
@@ -38,6 +70,8 @@ public RubyMatchData(RubyClass rubyClass, RubyBasicObject source, RubyRegexp reg
         this.pre = pre;
         this.post = post;
         this.global = global;
+        this.begin = begin;
+        this.end = end;
     }
 
     public Object[] getValues() {
@@ -51,31 +85,31 @@ public Object[] getCaptures() {
     }
 
     public Object begin(int index) {
-        if (region == null) {
-            throw new UnsupportedOperationException("begin is not yet working when no grouping data is available");
-        }
+        final int b = (region == null) ? begin : region.beg[index];
 
-        int begin = region.beg[index];
-
-        if (begin < 0) {
+        if (b < 0) {
             return getContext().getCoreLibrary().getNilObject();
         }
 
-        return begin;
+        updateCharOffset();
+
+        return charOffsets.beg[index];
     }
 
     public Object end(int index) {
-        if (region == null) {
-            throw new UnsupportedOperationException("end is not yet working when no grouping data is available");
-        }
-
-        int end = region.end[index];
+        int e = (region == null) ? end : region.end[index];
 
-        if (end < 0) {
+        if (e < 0) {
             return getContext().getCoreLibrary().getNilObject();
         }
 
-        return end;
+        final CodeRangeable sourceWrapped = StringNodes.getCodeRangeable(source);
+        if (!StringSupport.isSingleByteOptimizable(sourceWrapped, sourceWrapped.getByteList().getEncoding())) {
+            updateCharOffset();
+            e = charOffsets.end[index];
+        }
+
+        return e;
     }
 
     public int getNumberOfRegions() {
@@ -116,4 +150,142 @@ public RubyBasicObject getSource() {
     }
 
     public RubyBasicObject getRegexp() { return regexp; }
+
+    public Object getFullTuple() {
+        return fullTuple;
+    }
+
+    public void setFullTuple(Object fullTuple) {
+        this.fullTuple = fullTuple;
+    }
+
+    public int getFullBegin() {
+        return begin;
+    }
+
+    public int getFullEnd() {
+        return end;
+    }
+
+    // Taken from org.jruby.RubyMatchData.
+
+    private static final class Pair implements Comparable<Pair> {
+        int bytePos, charPos;
+        @Override
+        public int compareTo(Pair pair) {
+            return bytePos - pair.bytePos;
+        }
+    }
+
+    private void updatePairs(ByteList value, Encoding encoding, Pair[] pairs) {
+        Arrays.sort(pairs);
+
+        int length = pairs.length;
+        byte[]bytes = value.getUnsafeBytes();
+        int p = value.getBegin();
+        int s = p;
+        int c = 0;
+
+        for (int i = 0; i < length; i++) {
+            int q = s + pairs[i].bytePos;
+            c += StringSupport.strLength(encoding, bytes, p, q);
+            pairs[i].charPos = c;
+            p = q;
+        }
+    }
+
+
+    private void updateCharOffsetOnlyOneReg(ByteList value, Encoding encoding) {
+        if (charOffsetUpdated) return;
+
+        if (charOffsets == null || charOffsets.numRegs < 1) charOffsets = new Region(1);
+
+        if (encoding.maxLength() == 1) {
+            charOffsets.beg[0] = begin;
+            charOffsets.end[0] = end;
+            charOffsetUpdated = true;
+            return;
+        }
+
+        Pair[] pairs = new Pair[2];
+        if (begin >= 0) {
+            pairs[0] = new Pair();
+            pairs[0].bytePos = begin;
+            pairs[1] = new Pair();
+            pairs[1].bytePos = end;
+        }
+
+        updatePairs(value, encoding, pairs);
+
+        if (begin < 0) {
+            charOffsets.beg[0] = charOffsets.end[0] = -1;
+            return;
+        }
+        Pair key = new Pair();
+        key.bytePos = begin;
+        charOffsets.beg[0] = pairs[Arrays.binarySearch(pairs, key)].charPos;
+        key.bytePos = end;
+        charOffsets.end[0] = pairs[Arrays.binarySearch(pairs, key)].charPos;
+
+        charOffsetUpdated = true;
+    }
+
+    private void updateCharOffsetManyRegs(ByteList value, Encoding encoding) {
+        if (charOffsetUpdated) return;
+
+        final Region regs = this.region;
+        int numRegs = regs.numRegs;
+
+        if (charOffsets == null || charOffsets.numRegs < numRegs) charOffsets = new Region(numRegs);
+
+        if (encoding.maxLength() == 1) {
+            for (int i = 0; i < numRegs; i++) {
+                charOffsets.beg[i] = regs.beg[i];
+                charOffsets.end[i] = regs.end[i];
+            }
+            charOffsetUpdated = true;
+            return;
+        }
+
+        Pair[] pairs = new Pair[numRegs * 2];
+        for (int i = 0; i < pairs.length; i++) pairs[i] = new Pair();
+
+        int numPos = 0;
+        for (int i = 0; i < numRegs; i++) {
+            if (regs.beg[i] < 0) continue;
+            pairs[numPos++].bytePos = regs.beg[i];
+            pairs[numPos++].bytePos = regs.end[i];
+        }
+
+        updatePairs(value, encoding, pairs);
+
+        Pair key = new Pair();
+        for (int i = 0; i < regs.numRegs; i++) {
+            if (regs.beg[i] < 0) {
+                charOffsets.beg[i] = charOffsets.end[i] = -1;
+                continue;
+            }
+            key.bytePos = regs.beg[i];
+            charOffsets.beg[i] = pairs[Arrays.binarySearch(pairs, key)].charPos;
+            key.bytePos = regs.end[i];
+            charOffsets.end[i] = pairs[Arrays.binarySearch(pairs, key)].charPos;
+        }
+
+        charOffsetUpdated = true;
+    }
+
+    private void updateCharOffset() {
+        if (charOffsetUpdated) return;
+
+        ByteList value = StringNodes.getByteList(source);
+        Encoding enc = value.getEncoding();
+
+        if (region == null) {
+            updateCharOffsetOnlyOneReg(value, enc);
+        } else {
+            updateCharOffsetManyRegs(value, enc);
+        }
+
+        charOffsetUpdated = true;
+    }
 }
diff --git a/truffle/src/main/java/org/jruby/truffle/runtime/core/RubyRegexp.java b/truffle/src/main/java/org/jruby/truffle/runtime/core/RubyRegexp.java
@@ -107,7 +107,10 @@ public Object matchCommon(RubyString source, boolean operator, boolean setNamedC
         final byte[] stringBytes = StringNodes.getByteList(source).bytes();
 
         final ByteList bl = this.getSource();
-        final Regex r = new Regex(bl.getUnsafeBytes(), bl.getBegin(), bl.getBegin() + bl.getRealSize(), this.options.toJoniOptions(), checkEncoding(StringNodes.getCodeRangeable(source), true));
+        final Encoding enc = checkEncoding(StringNodes.getCodeRangeable(source), true);
+        final ByteList preprocessed = RegexpSupport.preprocess(getContext().getRuntime(), bl, enc, new Encoding[]{null}, RegexpSupport.ErrorMode.RAISE);
+
+        final Regex r = new Regex(preprocessed.getUnsafeBytes(), preprocessed.getBegin(), preprocessed.getBegin() + preprocessed.getRealSize(), this.options.toJoniOptions(), checkEncoding(StringNodes.getCodeRangeable(source), true));
         final Matcher matcher = r.matcher(stringBytes);
         int range = stringBytes.length;
 
@@ -169,7 +172,7 @@ public Object matchCommon(RubyString source, boolean operator, boolean setNamedC
         final RubyBasicObject post = makeString(source, region.end[0], bytes.length() - region.end[0]);
         final RubyBasicObject global = makeString(source, region.beg[0], region.end[0] - region.beg[0]);
 
-        final RubyMatchData matchObject = new RubyMatchData(context.getCoreLibrary().getMatchDataClass(), source, this, region, values, pre, post, global);
+        final RubyMatchData matchObject = new RubyMatchData(context.getCoreLibrary().getMatchDataClass(), source, this, region, values, pre, post, global, matcher.getBegin(), matcher.getEnd());
 
         if (operator) {
             if (values.length > 0) {