Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: bd313e0520db
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: eb8b2c99322f
Choose a head ref
  • 7 commits
  • 10 files changed
  • 1 contributor

Commits on Dec 24, 2016

  1. Copy the full SHA
    e5070c0 View commit details
  2. Copy the full SHA
    e37c0ad View commit details
  3. Copy the full SHA
    f1f13b9 View commit details
  4. Copy the full SHA
    7f2aa7c View commit details
  5. Copy the full SHA
    feb3824 View commit details
  6. Copy the full SHA
    35ff934 View commit details
  7. Copy the full SHA
    eb8b2c9 View commit details
22 changes: 0 additions & 22 deletions truffle/src/main/java/org/jruby/truffle/core/string/ByteList.java
Original file line number Diff line number Diff line change
@@ -50,7 +50,6 @@
public class ByteList implements CharSequence {

public static final byte[] NULL_ARRAY = new byte[0];
public static final ByteList EMPTY_BYTELIST = new ByteList(0);

private byte[] bytes;
private int begin;
@@ -237,19 +236,6 @@ public ByteList dup() {
return dup;
}

/**
* Create a new ByteList but do not array copy the byte backing store.
*
* @return a new ByteList with same backing store
*/
public ByteList shallowDup() {
ByteList dup = new ByteList(bytes, false);
dup.realSize = realSize;
dup.begin = begin;
dup.encoding = safeEncoding(encoding);
return dup;
}

/**
* @param length is the value of how big the buffer is going to be, not the actual length to copy
*
@@ -774,14 +760,6 @@ public byte[] getUnsafeBytes() {
return bytes;
}

/**
* @param bytes the bytes to set
*/
public void setUnsafeBytes(byte[] bytes) {
assert bytes != null;
this.bytes = bytes;
}

/**
* @return the begin
*/
114 changes: 114 additions & 0 deletions truffle/src/main/java/org/jruby/truffle/parser/ParserByteList.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package org.jruby.truffle.parser;

import org.jcodings.Encoding;
import org.jruby.truffle.core.string.ByteList;

public class ParserByteList {

private final ByteList byteList;

public ParserByteList(ByteList byteList) {
this.byteList = byteList;
}

public ParserByteList(int capacity) {
this(new ByteList(capacity));
}

public ParserByteList() {
this(new ByteList());
}

public ParserByteList(byte[] bytes, int start, int length, Encoding encoding, boolean copy) {
this(new ByteList(bytes, start, length, encoding, copy));
}

public ParserByteList(byte[] bytes, int start, int length, Encoding encoding) {
this(new ByteList(bytes, start, length, encoding, false));
}

public ParserByteList(byte[] bytes) {
this(new ByteList(bytes));
}

public ParserByteList(ParserByteList other) {
this(new ByteList(other.byteList));
}

public static ParserByteList create(String string) {
return new ParserByteList(ByteList.create(string));
}

public int getStart() {
return byteList.getBegin();
}

public void setStart(int start) {
byteList.setBegin(start);
}

public int getLength() {
return byteList.length();
}

public void setLength(int length) {
byteList.setRealSize(length);
}

public Encoding getEncoding() {
return byteList.getEncoding();
}

public void setEncoding(Encoding encoding) {
byteList.setEncoding(encoding);
}

public void append(int b) {
byteList.append(b);
}

public void append(byte[] bytes) {
byteList.append(bytes);
}

public void append(ParserByteList other, int start, int length) {
byteList.append(other.byteList, start, length);
}

public ParserByteList makeShared(int start, int length) {
return new ParserByteList(byteList.makeShared(start, length));
}

public byte[] getUnsafeBytes() {
return byteList.getUnsafeBytes();
}

public int caseInsensitiveCmp(ParserByteList other) {
return byteList.caseInsensitiveCmp(other.byteList);
}

public ByteList toByteList() {
return byteList;
}

public boolean equal(ParserByteList other) {
return byteList.equals(other.byteList);
}

public void ensure(int length) {
byteList.ensure(length);
}

public int charAt(int index) {
return byteList.charAt(index);
}

public void append(ParserByteList other) {
byteList.append(other.byteList);
}

public String toString() {
return byteList.toString();
}

}
Original file line number Diff line number Diff line change
@@ -30,12 +30,12 @@
package org.jruby.truffle.parser.lexer;

import org.jcodings.Encoding;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.parser.Tokens;

import static org.jruby.truffle.parser.lexer.LexingCommon.EOF;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_EXPAND;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_INDENT;
import static org.jruby.truffle.parser.lexer.RubyLexer.EOF;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_EXPAND;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_INDENT;

/**
* A lexing unit for scanning a heredoc element.
@@ -53,7 +53,7 @@
*/
public class HeredocTerm extends StrTerm {
// Marker delimiting heredoc boundary
private final ByteList nd_lit;
private final ParserByteList nd_lit;

// Expand variables, Indentation of final marker
private final int flags;
@@ -63,9 +63,9 @@ public class HeredocTerm extends StrTerm {
protected final int line;

// Portion of line right after beginning marker
protected final ByteList lastLine;
protected final ParserByteList lastLine;

public HeredocTerm(ByteList marker, int func, int nth, int line, ByteList lastLine) {
public HeredocTerm(ParserByteList marker, int func, int nth, int line, ParserByteList lastLine) {
this.nd_lit = marker;
this.flags = func;
this.nth = nth;
@@ -78,7 +78,7 @@ public int getFlags() {
return flags;
}

protected int error(RubyLexer lexer, int len, ByteList str, ByteList eos) {
protected int error(RubyLexer lexer, int len, ParserByteList str, ParserByteList eos) {
lexer.compile_error("can't find string \"" + eos.toString() + "\" anywhere before EOF");
return -1;
}
@@ -92,9 +92,9 @@ protected int restore(RubyLexer lexer) {

@Override
public int parseString(RubyLexer lexer) throws java.io.IOException {
ByteList str = null;
ByteList eos = nd_lit;
int len = nd_lit.length() - 1;
ParserByteList str = null;
ParserByteList eos = nd_lit;
int len = nd_lit.getLength() - 1;
boolean indent = (flags & STR_FUNC_INDENT) != 0;
int c = lexer.nextc();

@@ -108,7 +108,7 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {

if ((flags & STR_FUNC_EXPAND) == 0) {
do {
ByteList lbuf = lexer.lex_lastline;
ParserByteList lbuf = lexer.lex_lastline;
int p = 0;
int pend = lexer.lex_pend;
if (pend > p) {
@@ -134,7 +134,7 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
if (str != null) {
str.append(lbuf.makeShared(p, pend - p));
} else {
str = new ByteList(lbuf.makeShared(p, pend - p));
str = new ParserByteList(lbuf.makeShared(p, pend - p));
}

if (pend < lexer.lex_pend) str.append('\n');
@@ -148,7 +148,7 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
if (lexer.nextc() == -1) return error(lexer, len, null, eos);
} while (!lexer.whole_match_p(eos, indent));
} else {
ByteList tok = new ByteList();
ParserByteList tok = new ParserByteList();
tok.setEncoding(lexer.getEncoding());
if (c == '#') {
switch (c = lexer.nextc()) {
Original file line number Diff line number Diff line change
@@ -38,11 +38,9 @@

import com.oracle.truffle.api.source.Source;
import org.jcodings.Encoding;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;

import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

public class LexerSource {

@@ -54,25 +52,34 @@ public class LexerSource {

private int byteOffset;

private final List<ByteList> scriptLines = new ArrayList<>();

public LexerSource(Source source, int lineStartOffset, Encoding encoding) {
this.source = source;
this.lineStartOffset = lineStartOffset;
this.sourceBytes = source.getCode().getBytes(StandardCharsets.UTF_8);
this.encoding = encoding;
}

public Source getSource() {
return source;
}

public Encoding getEncoding() {
return encoding;
}

public void setEncoding(Encoding encoding) {
this.encoding = encoding;
scriptLines.forEach(b -> b.setEncoding(encoding));
}

public ByteList gets() {
public int getOffset() {
return byteOffset;
}

public int getLineStartOffset() {
return lineStartOffset;
}

public ParserByteList gets() {
if (byteOffset >= sourceBytes.length) {
return null;
}
@@ -83,15 +90,15 @@ public ByteList gets() {
lineEnd = sourceBytes.length;
}

final ByteList line = new ByteList(sourceBytes, byteOffset, lineEnd - byteOffset, encoding, false);
scriptLines.add(line);
final int start = byteOffset;
final int length = lineEnd - byteOffset;

byteOffset = lineEnd;

return line;
return new ParserByteList(sourceBytes, start, length, encoding);
}

public int nextNewLine() {
private int nextNewLine() {
int n = byteOffset;

while (n < sourceBytes.length) {
@@ -105,16 +112,4 @@ public int nextNewLine() {
return -1;
}

public Source getSource() {
return source;
}

public int getOffset() {
return byteOffset;
}

public int getLineOffset() {
return lineStartOffset;
}

}
812 changes: 0 additions & 812 deletions truffle/src/main/java/org/jruby/truffle/parser/lexer/LexingCommon.java

This file was deleted.

885 changes: 830 additions & 55 deletions truffle/src/main/java/org/jruby/truffle/parser/lexer/RubyLexer.java

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -29,21 +29,21 @@

import org.jcodings.Encoding;
import org.jruby.truffle.core.regexp.RegexpOptions;
import org.jruby.truffle.core.string.ByteList;
import org.jruby.truffle.parser.ParserByteList;
import org.jruby.truffle.parser.ast.RegexpParseNode;
import org.jruby.truffle.parser.parser.Tokens;
import org.jruby.truffle.core.string.KCode;

import java.io.IOException;

import static org.jruby.truffle.parser.lexer.LexingCommon.EOF;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_ESCAPE;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_EXPAND;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_QWORDS;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_REGEXP;
import static org.jruby.truffle.parser.lexer.LexingCommon.STR_FUNC_SYMBOL;
import static org.jruby.truffle.parser.lexer.LexingCommon.isHexChar;
import static org.jruby.truffle.parser.lexer.LexingCommon.isOctChar;
import static org.jruby.truffle.parser.lexer.RubyLexer.EOF;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_ESCAPE;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_EXPAND;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_QWORDS;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_REGEXP;
import static org.jruby.truffle.parser.lexer.RubyLexer.STR_FUNC_SYMBOL;
import static org.jruby.truffle.parser.lexer.RubyLexer.isHexChar;
import static org.jruby.truffle.parser.lexer.RubyLexer.isOctChar;

public class StringTerm extends StrTerm {
// Expand variables, Indentation of final marker
@@ -70,8 +70,8 @@ public int getFlags() {
return flags;
}

protected ByteList createByteList(RubyLexer lexer) {
ByteList bytelist = new ByteList(15);
protected ParserByteList createByteList(RubyLexer lexer) {
ParserByteList bytelist = new ParserByteList(15);
bytelist.setEncoding(lexer.getEncoding());
return bytelist;
}
@@ -85,9 +85,9 @@ private int endFound(RubyLexer lexer) {

if ((flags & STR_FUNC_REGEXP) != 0) {
RegexpOptions options = parseRegexpFlags(lexer);
ByteList regexpBytelist = ByteList.create("");
ParserByteList regexpBytelist = ParserByteList.create("");

lexer.setValue(new RegexpParseNode(lexer.getPosition(), regexpBytelist, options));
lexer.setValue(new RegexpParseNode(lexer.getPosition(), regexpBytelist.toByteList(), options));
return Tokens.tREGEXP_END;
}

@@ -191,8 +191,8 @@ public int parseString(RubyLexer lexer) throws IOException {
lexer.getPosition();
return ' ';
}
ByteList buffer = createByteList(lexer);

ParserByteList buffer = createByteList(lexer);
lexer.newtok(true);
if ((flags & STR_FUNC_EXPAND) != 0 && c == '#') {
int token = parsePeekVariableName(lexer);
@@ -266,7 +266,7 @@ private void mixedEscape(RubyLexer lexer, Encoding foundEncoding, Encoding parse
}

// mri: parser_tokadd_string
public int parseStringIntoBuffer(RubyLexer lexer, ByteList buffer, Encoding enc[]) throws IOException {
public int parseStringIntoBuffer(RubyLexer lexer, ParserByteList buffer, Encoding enc[]) throws IOException {
boolean qwords = (flags & STR_FUNC_QWORDS) != 0;
boolean expand = (flags & STR_FUNC_EXPAND) != 0;
boolean escape = (flags & STR_FUNC_ESCAPE) != 0;
@@ -420,7 +420,7 @@ private boolean simple_re_meta(int c) {

// Was a goto in original ruby lexer
@SuppressWarnings("fallthrough")
private void escaped(RubyLexer lexer, ByteList buffer) throws IOException {
private void escaped(RubyLexer lexer, ParserByteList buffer) throws IOException {
int c;

switch (c = lexer.nextc()) {
@@ -435,7 +435,7 @@ private void escaped(RubyLexer lexer, ByteList buffer) throws IOException {
}

@SuppressWarnings("fallthrough")
private void parseEscapeIntoBuffer(RubyLexer lexer, ByteList buffer) throws IOException {
private void parseEscapeIntoBuffer(RubyLexer lexer, ParserByteList buffer) throws IOException {
int c;

switch (c = lexer.nextc()) {
Original file line number Diff line number Diff line change
@@ -137,9 +137,9 @@
import java.util.ArrayList;
import java.util.List;

import static org.jruby.truffle.parser.lexer.LexingCommon.ASCII8BIT_ENCODING;
import static org.jruby.truffle.parser.lexer.LexingCommon.USASCII_ENCODING;
import static org.jruby.truffle.parser.lexer.LexingCommon.UTF8_ENCODING;
import static org.jruby.truffle.parser.lexer.RubyLexer.ASCII8BIT_ENCODING;
import static org.jruby.truffle.parser.lexer.RubyLexer.USASCII_ENCODING;
import static org.jruby.truffle.parser.lexer.RubyLexer.UTF8_ENCODING;

/**
*
Original file line number Diff line number Diff line change
@@ -132,12 +132,12 @@

import java.io.IOException;

import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_BEG;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_END;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_ENDARG;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_ENDFN;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_FNAME;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_LABEL;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_BEG;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_END;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_ENDARG;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_ENDFN;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_FNAME;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_LABEL;

@SuppressWarnings({"unchecked", "fallthrough"})
public class RubyParser {
@@ -2565,7 +2565,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[281] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = Long.valueOf(lexer.getCmdArgumentState().getStack());
yyVal = lexer.getCmdArgumentState().getStack();
lexer.getCmdArgumentState().begin();
return yyVal;
}
Original file line number Diff line number Diff line change
@@ -129,12 +129,12 @@ import org.jruby.truffle.parser.lexer.SyntaxException.PID;

import java.io.IOException;

import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_BEG;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_END;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_ENDARG;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_ENDFN;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_FNAME;
import static org.jruby.truffle.parser.lexer.LexingCommon.EXPR_LABEL;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_BEG;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_END;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_ENDARG;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_ENDFN;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_FNAME;
import static org.jruby.truffle.parser.lexer.RubyLexer.EXPR_LABEL;

@SuppressWarnings({"unchecked", "fallthrough"})
public class RubyParser {
@@ -1248,7 +1248,7 @@ call_args : command {
}

command_args : /* none */ {
$$ = Long.valueOf(lexer.getCmdArgumentState().getStack());
$$ = lexer.getCmdArgumentState().getStack();
lexer.getCmdArgumentState().begin();
} call_args {
lexer.getCmdArgumentState().reset($<Long>1.longValue());