Skip to content

Commit

Permalink
Showing 15 changed files with 98 additions and 101 deletions.
Original file line number Diff line number Diff line change
@@ -38,10 +38,12 @@
import org.jruby.truffle.core.string.StringUtils;
import org.jruby.truffle.language.RubyGuards;
import org.jruby.truffle.collections.Memo;
import org.jruby.truffle.parser.ParserByteList;

import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Deque;
import java.util.concurrent.ConcurrentHashMap;

@@ -639,6 +641,10 @@ public static Rope ropeFromByteList(ByteList byteList) {
return create(byteList.bytes(), byteList.getEncoding(), CR_UNKNOWN);
}

public static Rope ropeFromByteList(ParserByteList byteList) {
return create(Arrays.copyOfRange(byteList.getUnsafeBytes(), byteList.getStart(), byteList.getStart() + byteList.getLength()), byteList.getEncoding(), CR_UNKNOWN);
}

public static Rope ropeFromByteList(ByteList byteList, CodeRange codeRange) {
// TODO (nirvdrum 08-Jan-16) We need to make a copy of the ByteList's bytes for now to be safe, but we should be able to use the unsafe bytes as we move forward.
return create(byteList.bytes(), byteList.getEncoding(), codeRange);
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
import org.jcodings.Encoding;
import org.jcodings.specific.UTF8Encoding;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.parser.ParserByteList;

import java.lang.ref.WeakReference;
import java.util.Arrays;
@@ -38,6 +39,10 @@ public Rope getRopeUTF8(String string) {
return getRope(string);
}

public Rope getRope(ParserByteList string, CodeRange codeRange) {
return getRope(Arrays.copyOfRange(string.getUnsafeBytes(), string.getStart(), string.getStart() + string.getLength()), string.getEncoding(), codeRange);
}

@TruffleBoundary
public Rope getRope(String string) {
lock.readLock().lock();
Original file line number Diff line number Diff line change
@@ -36,6 +36,7 @@
import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.rope.Rope;
import org.jruby.truffle.core.rope.RopeOperations;
import org.jruby.truffle.parser.ParserByteList;

import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -191,6 +192,10 @@ private static CodeRange codeRangeScanNonAsciiCompatible(Encoding enc, byte[]byt
return p > end ? CR_BROKEN : CR_VALID;
}

public static CodeRange codeRangeScan(Encoding enc, ParserByteList bytes) {
return codeRangeScan(enc, bytes.getUnsafeBytes(), bytes.getStart(), bytes.getLength());
}

public static CodeRange codeRangeScan(Encoding enc, ByteList bytes) {
return codeRangeScan(enc, bytes.getUnsafeBytes(), bytes.getBegin(), bytes.getRealSize());
}
Original file line number Diff line number Diff line change
@@ -62,7 +62,6 @@
import org.jruby.truffle.core.rope.RopeOperations;
import org.jruby.truffle.core.rubinius.RubiniusLastStringReadNode;
import org.jruby.truffle.core.rubinius.RubiniusLastStringWriteNodeGen;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.core.string.InterpolatedStringNode;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.core.string.StringUtils;
@@ -511,10 +510,10 @@ public RubyNode visitCallNode(CallParseNode node) {

if (receiver instanceof StrParseNode && methodName.equals("freeze")) {
final StrParseNode strNode = (StrParseNode) receiver;
final ByteList byteList = strNode.getValue();
final ParserByteList byteList = strNode.getValue();
final CodeRange codeRange = strNode.getCodeRange();

final Rope rope = context.getRopeTable().getRope(byteList.bytes(), byteList.getEncoding(), codeRange);
final Rope rope = context.getRopeTable().getRope(byteList, codeRange);

final DynamicObject frozenString = context.getFrozenStrings().getFrozenString(rope);

@@ -2122,7 +2121,7 @@ public RubyNode visitMatch2Node(Match2ParseNode node) {

if (node.getReceiverNode() instanceof RegexpParseNode) {
final RegexpParseNode regexpNode = (RegexpParseNode) node.getReceiverNode();
final Regex regex = new Regex(regexpNode.getValue().bytes(), 0, regexpNode.getValue().length(), regexpNode.getOptions().toOptions(), regexpNode.getEncoding(), Syntax.RUBY);
final Regex regex = new Regex(regexpNode.getValue().getUnsafeBytes(), regexpNode.getValue().getStart(), regexpNode.getValue().getStart() + regexpNode.getValue().getLength(), regexpNode.getOptions().toOptions(), regexpNode.getEncoding(), Syntax.RUBY);

if (regex.numberOfNames() > 0) {
for (Iterator<NameEntry> i = regex.namedBackrefIterator(); i.hasNext(); ) {
@@ -3089,9 +3088,9 @@ public RubyNode visitSplatNode(SplatParseNode node) {
public RubyNode visitStrNode(StrParseNode node) {
final TempSourceSection sourceSection = node.getPosition();

final ByteList byteList = node.getValue();
final ParserByteList byteList = node.getValue();
final CodeRange codeRange = node.getCodeRange();
final Rope rope = context.getRopeTable().getRope(byteList.bytes(), byteList.getEncoding(), codeRange);
final Rope rope = context.getRopeTable().getRope(byteList, codeRange);

final RubyNode ret;

Original file line number Diff line number Diff line change
@@ -41,8 +41,8 @@
import org.jcodings.ascii.AsciiTables;
import org.jcodings.specific.ASCIIEncoding;

import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;

public class ParserByteList {

@@ -133,8 +133,9 @@ public int charAt(int index) {
return bytes[start + index];
}

@Override
public String toString() {
return new String(Arrays.copyOfRange(bytes, start, start + length), StandardCharsets.US_ASCII);
return StandardCharsets.ISO_8859_1.decode(ByteBuffer.wrap(bytes, start, length)).toString();
}

public byte[] getUnsafeBytes() {
Original file line number Diff line number Diff line change
@@ -80,4 +80,8 @@ public ParserByteList toParserByteList() {
return new ParserByteList(Arrays.copyOf(bytes, length), 0, length, encoding);
}

public void removeOffset(int offset) {
System.arraycopy(bytes, offset, bytes, 0, length - offset);
length -= offset;
}
}
Original file line number Diff line number Diff line change
@@ -28,14 +28,14 @@
***** END LICENSE BLOCK *****/
package org.jruby.truffle.parser.ast;

import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.TempSourceSection;

/**
* Represents __FILE__ nodes
*/
public class FileParseNode extends StrParseNode implements SideEffectFree {
public FileParseNode(TempSourceSection position, ByteList value) {
public FileParseNode(TempSourceSection position, ParserByteList value) {
super(position, value);
}
}
Original file line number Diff line number Diff line change
@@ -33,7 +33,6 @@

import org.jcodings.Encoding;
import org.jruby.truffle.core.regexp.RegexpOptions;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ast.types.ILiteralNode;
import org.jruby.truffle.parser.ast.visitor.NodeVisitor;
@@ -45,14 +44,10 @@
* Represents a simple regular expression literal.
*/
public class RegexpParseNode extends ParseNode implements ILiteralNode {
private final ByteList value;
private final ParserByteList value;
private final RegexpOptions options;

public RegexpParseNode(TempSourceSection position, ParserByteList value, RegexpOptions options) {
this(position, new ByteList(value.getUnsafeBytes(), value.getStart(), value.getLength(), value.getEncoding(), false), options);
}

public RegexpParseNode(TempSourceSection position, ByteList value, RegexpOptions options) {
super(position, false);

this.value = value;
@@ -80,9 +75,9 @@ public RegexpOptions getOptions() {

/**
* Gets the value.
* @return Returns a ByteList
* @return Returns a ParserByteList
*/
public ByteList getValue() {
public ParserByteList getValue() {
return value;
}

Original file line number Diff line number Diff line change
@@ -33,9 +33,9 @@
package org.jruby.truffle.parser.ast;

import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.core.string.StringSupport;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ParserByteListBuilder;
import org.jruby.truffle.parser.TempSourceSection;
import org.jruby.truffle.parser.ast.types.ILiteralNode;
import org.jruby.truffle.parser.ast.visitor.NodeVisitor;
@@ -46,23 +46,15 @@
* Representing a simple String literal.
*/
public class StrParseNode extends ParseNode implements ILiteralNode, SideEffectFree {
private final ByteList value;
private ParserByteList value;
private final CodeRange codeRange;
private boolean frozen;

public StrParseNode(TempSourceSection position, ParserByteList value) {
this(position, new ByteList(value.getUnsafeBytes(), value.getStart(), value.getLength(), value.getEncoding(), false));
}

public StrParseNode(TempSourceSection position, ByteList value) {
this(position, value, StringSupport.codeRangeScan(value.getEncoding(), value));
}

public StrParseNode(TempSourceSection position, ParserByteList value, CodeRange codeRange) {
this(position, new ByteList(value.getUnsafeBytes(), value.getStart(), value.getLength(), value.getEncoding(), false));
}

public StrParseNode(TempSourceSection position, ByteList value, CodeRange codeRange) {
super(position, false);

this.value = value;
@@ -72,16 +64,16 @@ public StrParseNode(TempSourceSection position, ByteList value, CodeRange codeRa
public StrParseNode(TempSourceSection position, StrParseNode head, StrParseNode tail) {
super(position, false);

ByteList headBL = head.getValue();
ByteList tailBL = tail.getValue();
ParserByteList headBL = head.getValue();
ParserByteList tailBL = tail.getValue();

ByteList myValue = new ByteList(headBL.getRealSize() + tailBL.getRealSize());
ParserByteListBuilder myValue = new ParserByteListBuilder();
myValue.setEncoding(headBL.getEncoding());
myValue.append(headBL);
myValue.append(tailBL);

frozen = head.isFrozen() && tail.isFrozen();
value = myValue;
value = myValue.toParserByteList();
codeRange = StringSupport.codeRangeScan(value.getEncoding(), value);
}

@@ -100,7 +92,7 @@ public <T> T accept(NodeVisitor<T> iVisitor) {
* Gets the value.
* @return Returns a String
*/
public ByteList getValue() {
public ParserByteList getValue() {
return value;
}

@@ -124,4 +116,8 @@ public boolean isFrozen() {
public void setFrozen(boolean frozen) {
this.frozen = frozen;
}

public void setValue(ParserByteList value) {
this.value = value;
}
}
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@
import org.jcodings.Encoding;
import org.jcodings.specific.USASCIIEncoding;
import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ast.types.ILiteralNode;
import org.jruby.truffle.parser.ast.types.INameNode;
import org.jruby.truffle.parser.ast.visitor.NodeVisitor;
@@ -63,13 +63,13 @@ public SymbolParseNode(TempSourceSection position, String name, Encoding encodin
}

// String path (e.g. [':', str_beg, str_content, str_end])
public SymbolParseNode(TempSourceSection position, ByteList value) {
public SymbolParseNode(TempSourceSection position, ParserByteList value) {
super(position, false);
this.name = value.toString().intern();

if (value.getEncoding() != USASCIIEncoding.INSTANCE) {
int size = value.realSize();
this.encoding = value.getEncoding().strLength(value.unsafeBytes(), value.begin(), size) == size ?
int size = value.getLength();
this.encoding = value.getEncoding().strLength(value.getUnsafeBytes(), value.getStart(), size) == size ?
USASCIIEncoding.INSTANCE : value.getEncoding();
} else {
this.encoding = USASCIIEncoding.INSTANCE;
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@
package org.jruby.truffle.parser.ast;

import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ast.types.ILiteralNode;
import org.jruby.truffle.parser.ast.visitor.NodeVisitor;
import org.jruby.truffle.parser.TempSourceSection;
@@ -43,13 +43,13 @@
* A Backtick(`) string
*/
public class XStrParseNode extends ParseNode implements ILiteralNode {
private final ByteList value;
private final ParserByteList value;
private CodeRange coderange;

public XStrParseNode(TempSourceSection position, ByteList value, CodeRange coderange) {
public XStrParseNode(TempSourceSection position, ParserByteList value, CodeRange coderange) {
// FIXME: Shouldn't this have codeRange like StrParseNode?
super(position, false);
this.value = (value == null ? ByteList.create("") : value);
this.value = (value == null ? new ParserByteList(new byte[]{}) : value);
this.coderange = coderange;
}

@@ -69,7 +69,7 @@ public <T> T accept(NodeVisitor<T> iVisitor) {
* Gets the value.
* @return Returns a String
*/
public ByteList getValue() {
public ParserByteList getValue() {
return value;
}

27 changes: 15 additions & 12 deletions truffle/src/main/java/org/jruby/truffle/parser/lexer/RubyLexer.java
Original file line number Diff line number Diff line change
@@ -54,7 +54,6 @@
import org.jruby.truffle.RubyContext;
import org.jruby.truffle.core.regexp.ClassicRegexp;
import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.core.string.StringSupport;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ParserByteListBuilder;
@@ -313,7 +312,10 @@ public void heredoc_dedent(ParseNode root) {

if (root instanceof StrParseNode) {
StrParseNode str = (StrParseNode) root;
dedent_string(str.getValue(), indent);
ParserByteListBuilder builder = new ParserByteListBuilder();
builder.append(str.getValue());
dedent_string(builder, indent);
str.setValue(builder.toParserByteList());
} else if (root instanceof ListParseNode) {
ListParseNode list = (ListParseNode) root;
int length = list.size();
@@ -325,7 +327,10 @@ public void heredoc_dedent(ParseNode root) {
currentLine = child.getPosition().getStartLine() - 1; // New line

if (child instanceof StrParseNode) {
dedent_string(((StrParseNode) child).getValue(), indent);
ParserByteListBuilder builder = new ParserByteListBuilder();
builder.append(((StrParseNode) child).getValue());
dedent_string(builder, indent);
((StrParseNode) child).setValue(builder.toParserByteList());
}
}
}
@@ -2649,16 +2654,15 @@ public String createTokenString() {
return createTokenString(tokp);
}

protected int dedent_string(ByteList string, int width) {
long len = string.realSize();
protected int dedent_string(ParserByteListBuilder string, int width) {
long len = string.getLength();
int i, col = 0;
byte[] str = string.unsafeBytes();
int begin = string.begin();
byte[] str = string.getUnsafeBytes();

for (i = 0; i < len && col < width; i++) {
if (str[begin + i] == ' ') {
if (str[i] == ' ') {
col++;
} else if (str[begin + i] == '\t') {
} else if (str[i] == '\t') {
int n = TAB_WIDTH * (col / TAB_WIDTH + 1);
if (n > width) break;
col = n;
@@ -2667,8 +2671,7 @@ protected int dedent_string(ByteList string, int width) {
}
}

string.setBegin(begin + i);
string.setRealSize((int) len - i);
string.removeOffset(i);
return i;
}

@@ -3314,4 +3317,4 @@ public static int magicCommentMarker(ParserByteList str, int begin) {
public static final Regex magicRegexp = new Regex(magicString.getBytes(), 0, magicString.length(), 0, Encoding.load("ASCII"));


}
}
Loading

0 comments on commit 9a1b231

Please sign in to comment.