Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 8bd21ecd4198
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: b32c9ddc841c
Choose a head ref
  • 6 commits
  • 8 files changed
  • 1 contributor

Commits on Jul 26, 2017

  1. Start moving symbol creation back to lexers: This commit iIDENTIFIER.

    @enebo
    enebo committed Jul 26, 2017
    Copy the full SHA
    f8bb714 View commit details

  2. Constants as symbols back to lexer

    @enebo
    enebo committed Jul 26, 2017
    Copy the full SHA
    7759167 View commit details

  3. tFID as Symbol in lexer.

    @enebo
    enebo committed Jul 26, 2017
    Copy the full SHA
    3997154 View commit details

  4. Make gvars symbols in lexer.

    @enebo
    enebo committed Jul 26, 2017
    Copy the full SHA
    36ad64e View commit details

  5. whoooooo closer to a symboltacracy in the lexer

    @enebo
    enebo committed Jul 26, 2017
    Copy the full SHA
    60536ee View commit details

Commits on Jul 27, 2017

  1. Disable copy of creating tokens since all ident paths are now made in… 

    …to symbols.

The earlier weird failure where something would mutate the backing store no
longer is happening.
    @enebo
    enebo committed Jul 27, 2017
    Copy the full SHA
    b32c9dd View commit details
1 change: 1 addition & 0 deletions core/src/main/java/org/jruby/ast/BackRefNode.java
View file
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@
import java.util.List;

import org.jcodings.specific.USASCIIEncoding;
import org.jruby.RubySymbol;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.lexer.yacc.ISourcePosition;
import org.jruby.util.ByteList;
17 changes: 6 additions & 11 deletions core/src/main/java/org/jruby/ast/VAliasNode.java
View file
Original file line number Diff line number Diff line change
@@ -33,29 +33,24 @@

import java.util.List;

import org.jruby.RubySymbol;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.lexer.yacc.ISourcePosition;
import org.jruby.util.ByteList;
import org.jruby.util.StringSupport;

/**
* Represents an alias of a global variable.
*/
public class VAliasNode extends Node {
private ByteList oldName;
private ByteList newName;
private RubySymbol oldName;
private RubySymbol newName;

public VAliasNode(ISourcePosition position, ByteList newName, ByteList oldName) {
public VAliasNode(ISourcePosition position, RubySymbol newName, RubySymbol oldName) {
super(position, false);
this.oldName = oldName;
this.newName = newName;
}

@Deprecated
public VAliasNode(ISourcePosition position, String newName, String oldName) {
this(position, StringSupport.stringAsByteList(newName), StringSupport.stringAsByteList(oldName));
}

public NodeType getNodeType() {
return NodeType.VALIASNODE;
}
@@ -73,15 +68,15 @@ public <T> T accept(NodeVisitor<T> iVisitor) {
* @return Returns a String
*/
public String getNewName() {
return StringSupport.byteListAsString(newName);
return StringSupport.byteListAsString(newName.getBytes());
}

/**
* Gets the oldName.
* @return Returns a String
*/
public String getOldName() {
return StringSupport.byteListAsString(oldName);
return StringSupport.byteListAsString(oldName.getBytes());
}

public List<Node> childNodes() {
2 changes: 1 addition & 1 deletion core/src/main/java/org/jruby/lexer/LexingCommon.java
View file
Original file line number Diff line number Diff line change
@@ -160,7 +160,7 @@ public void setRubySourceline(int line) {
}

public ByteList createTokenByteList() {
return new ByteList(lexb.unsafeBytes(), lexb.begin() + tokp, lex_p - tokp, getEncoding(), true);
return new ByteList(lexb.unsafeBytes(), lexb.begin() + tokp, lex_p - tokp, getEncoding(), false);
}

public ByteList createTokenByteList(int start) {
58 changes: 33 additions & 25 deletions core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
View file
Original file line number Diff line number Diff line change
@@ -67,6 +67,10 @@
import org.jruby.util.StringSupport;
import org.jruby.util.cli.Options;

import static org.jruby.parser.RubyParser.tCONSTANT;
import static org.jruby.parser.RubyParser.tFID;
import static org.jruby.parser.RubyParser.tIDENTIFIER;

/*
* This is a port of the MRI lexer to Java.
*/
@@ -149,6 +153,10 @@ protected void ambiguousOperator(String op, String syn) {
warnings.warn(ID.AMBIGUOUS_ARGUMENT, getFile(), ruby_sourceline, "even though it seems like " + syn);
}

private RubySymbol createTokenSymbol() {
return parserSupport.symbol(createTokenByteList());
}

public enum Keyword {
END ("end", new ByteList(new byte[] {'e', 'n', 'd'}, USASCII_ENCODING), RubyParser.keyword_end, RubyParser.keyword_end, EXPR_END),
ELSE ("else", new ByteList(new byte[] {'e', 'l', 's', 'e'}, USASCII_ENCODING), RubyParser.keyword_else, RubyParser.keyword_else, EXPR_BEG),
@@ -272,14 +280,13 @@ public static Keyword getKeyword(String str) {

public int tokenize_ident(int result) {
// FIXME: Get token from newtok index to lex_p?
ByteList value = createTokenByteList();
RubySymbol symbol = parserSupport.symbol(value);
RubySymbol symbol = createTokenSymbol();

if (isLexState(last_state, EXPR_DOT|EXPR_FNAME) && parserSupport.getCurrentScope().isDefined(symbol) >= 0) {
setState(EXPR_END);
}

yaccValue = value;
yaccValue = symbol;
return result;
}

@@ -745,11 +752,11 @@ private void printToken(int token) {
switch (token) {
case RubyParser.yyErrorCode: System.err.print("yyErrorCode,"); break;
// MISSING tokens
case RubyParser.tIDENTIFIER: System.err.print("tIDENTIFIER["+ value() + "],"); break;
case RubyParser.tFID: System.err.print("tFID[" + value() + "],"); break;
case tIDENTIFIER: System.err.print("tIDENTIFIER["+ value() + "],"); break;
case tFID: System.err.print("tFID[" + value() + "],"); break;
case RubyParser.tGVAR: System.err.print("tGVAR[" + value() + "],"); break;
case RubyParser.tIVAR: System.err.print("tIVAR[" + value() +"],"); break;
case RubyParser.tCONSTANT: System.err.print("tCONSTANT["+ value() +"],"); break;
case tCONSTANT: System.err.print("tCONSTANT["+ value() +"],"); break;
case RubyParser.tCVAR: System.err.print("tCVAR,"); break;
case RubyParser.tINTEGER: System.err.print("tINTEGER,"); break;
case RubyParser.tFLOAT: System.err.print("tFLOAT,"); break;
@@ -1078,13 +1085,14 @@ private int yylex() throws IOException {
}
}

private int identifierToken(int result, ByteList value) {
if (result == RubyParser.tIDENTIFIER && !isLexState(last_state, EXPR_DOT|EXPR_FNAME) &&
parserSupport.getCurrentScope().isDefined(parserSupport.symbol(value)) >= 0) {
private int identifierToken(int result, RubySymbol value) {
if (result == tIDENTIFIER && !isLexState(last_state, EXPR_DOT|EXPR_FNAME) &&
parserSupport.getCurrentScope().isDefined(value) >= 0) {
setState(EXPR_END|EXPR_LABEL);
}

yaccValue = value;

return result;
}

@@ -1307,7 +1315,7 @@ private int dollar() throws IOException {

last_state = lex_state;
setState(EXPR_END);
yaccValue = createTokenByteList();
yaccValue = createTokenSymbol();
return RubyParser.tGVAR;

}
@@ -1330,7 +1338,7 @@ private int dollar() throws IOException {
case '<': /* $<: reading filename */
case '>': /* $>: default output handle */
case '\"': /* $": already loaded files */
yaccValue = new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING);
yaccValue = parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING));
return RubyParser.tGVAR;


@@ -1343,7 +1351,7 @@ private int dollar() throws IOException {
pushback('-');
return '$';
}
yaccValue = createTokenByteList();
yaccValue = createTokenSymbol();
/* xxx shouldn't check if valid option variable */
return RubyParser.tGVAR;

@@ -1353,7 +1361,7 @@ private int dollar() throws IOException {
case '+': /* $+: string matches last paren. */
// Explicit reference to these vars as symbols...
if (isLexState(last_state, EXPR_FNAME)) {
yaccValue = new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING);
yaccValue = parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING));
return RubyParser.tGVAR;
}

@@ -1367,7 +1375,7 @@ private int dollar() throws IOException {
} while (Character.isDigit(c));
pushback(c);
if (isLexState(last_state, EXPR_FNAME)) {
yaccValue = createTokenByteList();
yaccValue = createTokenSymbol();
return RubyParser.tGVAR;
}

@@ -1386,7 +1394,7 @@ private int dollar() throws IOException {
case '0':
setState(EXPR_END);

return identifierToken(RubyParser.tGVAR, new ByteList(new byte[] {'$', (byte) c}));
return identifierToken(RubyParser.tGVAR, parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c})));
default:
if (!isIdentifierChar(c)) {
if (c == EOF || Character.isSpaceChar(c)) {
@@ -1402,7 +1410,7 @@ private int dollar() throws IOException {

tokadd_ident(c);

return identifierToken(RubyParser.tGVAR, createTokenByteList()); // $blah
return identifierToken(RubyParser.tGVAR, createTokenSymbol()); // $blah
}
}

@@ -1491,18 +1499,18 @@ private int identifier(int c, boolean commandState) throws IOException {
int result = 0;

last_state = lex_state;
ByteList tempVal;
RubySymbol tempVal;
if (lastBangOrPredicate) {
result = RubyParser.tFID;
tempVal = createTokenByteList();
result = tFID;
tempVal = createTokenSymbol();
} else {
if (isLexState(lex_state, EXPR_FNAME)) {
if ((c = nextc()) == '=') {
int c2 = nextc();

if (c2 != '~' && c2 != '>' &&
(c2 != '=' || peek('>'))) {
result = RubyParser.tIDENTIFIER;
result = tIDENTIFIER;
pushback(c2);
} else {
pushback(c2);
@@ -1512,12 +1520,12 @@ private int identifier(int c, boolean commandState) throws IOException {
pushback(c);
}
}
tempVal = createTokenByteList();
tempVal = createTokenSymbol();

if (result == 0 && Character.isUpperCase(StringSupport.preciseCodePoint(getEncoding(), tempVal.unsafeBytes(), tempVal.begin(), tempVal.begin() + 1))) {
result = RubyParser.tCONSTANT;
if (result == 0 && Character.isUpperCase(StringSupport.preciseCodePoint(getEncoding(), tempVal.getBytes().unsafeBytes(), tempVal.getBytes().begin(), tempVal.getBytes().begin() + 1))) {
result = tCONSTANT;
} else {
result = RubyParser.tIDENTIFIER;
result = tIDENTIFIER;
}
}

@@ -1531,7 +1539,7 @@ private int identifier(int c, boolean commandState) throws IOException {
}

if (lex_state != EXPR_DOT) {
Keyword keyword = getKeyword(tempVal); // Is it is a keyword?
Keyword keyword = getKeyword(tempVal.getBytes()); // Is it is a keyword?

if (keyword != null) {
int state = lex_state; // Save state at time keyword is encountered
5 changes: 5 additions & 0 deletions core/src/main/java/org/jruby/parser/ParserSupport.java
View file
Original file line number Diff line number Diff line change
@@ -1113,6 +1113,11 @@ public KeyValuePair<Node, Node> createKeyValue(Node key, Node value) {
return new KeyValuePair<>(key, value);
}

public Node asSymbol(ISourcePosition position, RubySymbol value) {
return new SymbolNode(position, value);
}

@Deprecated
public Node asSymbol(ISourcePosition position, ByteList value) {
return new SymbolNode(position, symbol(value));
}
Loading