jruby · Jul 26, 2017 · Jul 26, 2017 · Jul 26, 2017 · Jul 26, 2017 · Jul 26, 2017
diff --git a/core/src/main/java/org/jruby/ast/BackRefNode.java b/core/src/main/java/org/jruby/ast/BackRefNode.java
@@ -34,6 +34,7 @@
 import java.util.List;
 
 import org.jcodings.specific.USASCIIEncoding;
+import org.jruby.RubySymbol;
 import org.jruby.ast.visitor.NodeVisitor;
 import org.jruby.lexer.yacc.ISourcePosition;
 import org.jruby.util.ByteList;

diff --git a/core/src/main/java/org/jruby/ast/VAliasNode.java b/core/src/main/java/org/jruby/ast/VAliasNode.java
@@ -33,29 +33,24 @@
 
 import java.util.List;
 
+import org.jruby.RubySymbol;
 import org.jruby.ast.visitor.NodeVisitor;
 import org.jruby.lexer.yacc.ISourcePosition;
-import org.jruby.util.ByteList;
 import org.jruby.util.StringSupport;
 
 /** 
  * Represents an alias of a global variable.
  */
 public class VAliasNode extends Node {
-    private ByteList oldName;
-    private ByteList newName;
+    private RubySymbol oldName;
+    private RubySymbol newName;
 
-    public VAliasNode(ISourcePosition position, ByteList newName, ByteList oldName) {
+    public VAliasNode(ISourcePosition position, RubySymbol newName, RubySymbol oldName) {
         super(position, false);
         this.oldName = oldName;
         this.newName = newName;
     }
 
-    @Deprecated
-    public VAliasNode(ISourcePosition position, String newName, String oldName) {
-        this(position, StringSupport.stringAsByteList(newName), StringSupport.stringAsByteList(oldName));
-    }
-
     public NodeType getNodeType() {
         return NodeType.VALIASNODE;
     }
@@ -73,15 +68,15 @@ public <T> T accept(NodeVisitor<T> iVisitor) {
      * @return Returns a String
      */
     public String getNewName() {
-        return StringSupport.byteListAsString(newName);
+        return StringSupport.byteListAsString(newName.getBytes());
     }
 
     /**
      * Gets the oldName.
      * @return Returns a String
      */
     public String getOldName() {
-        return StringSupport.byteListAsString(oldName);
+        return StringSupport.byteListAsString(oldName.getBytes());
     }
 
     public List<Node> childNodes() {

diff --git a/core/src/main/java/org/jruby/lexer/LexingCommon.java b/core/src/main/java/org/jruby/lexer/LexingCommon.java
@@ -160,7 +160,7 @@ public void setRubySourceline(int line) {
     }
 
     public ByteList createTokenByteList() {
-        return new ByteList(lexb.unsafeBytes(), lexb.begin() + tokp, lex_p - tokp, getEncoding(), true);
+        return new ByteList(lexb.unsafeBytes(), lexb.begin() + tokp, lex_p - tokp, getEncoding(), false);
     }
 
     public ByteList createTokenByteList(int start) {

diff --git a/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java b/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
@@ -67,6 +67,10 @@
 import org.jruby.util.StringSupport;
 import org.jruby.util.cli.Options;
 
+import static org.jruby.parser.RubyParser.tCONSTANT;
+import static org.jruby.parser.RubyParser.tFID;
+import static org.jruby.parser.RubyParser.tIDENTIFIER;
+
 /*
  * This is a port of the MRI lexer to Java.
  */
@@ -149,6 +153,10 @@ protected void ambiguousOperator(String op, String syn) {
         warnings.warn(ID.AMBIGUOUS_ARGUMENT, getFile(), ruby_sourceline, "even though it seems like " + syn);
     }
 
+    private RubySymbol createTokenSymbol() {
+        return parserSupport.symbol(createTokenByteList());
+    }
+
     public enum Keyword {
         END ("end", new ByteList(new byte[] {'e', 'n', 'd'}, USASCII_ENCODING), RubyParser.keyword_end, RubyParser.keyword_end, EXPR_END),
         ELSE ("else", new ByteList(new byte[] {'e', 'l', 's', 'e'}, USASCII_ENCODING), RubyParser.keyword_else, RubyParser.keyword_else, EXPR_BEG),
@@ -272,14 +280,13 @@ public static Keyword getKeyword(String str) {
 
     public int tokenize_ident(int result) {
         // FIXME: Get token from newtok index to lex_p?
-        ByteList value = createTokenByteList();
-        RubySymbol symbol = parserSupport.symbol(value);
+        RubySymbol symbol = createTokenSymbol();
 
         if (isLexState(last_state, EXPR_DOT|EXPR_FNAME) && parserSupport.getCurrentScope().isDefined(symbol) >= 0) {
             setState(EXPR_END);
         }
 
-        yaccValue = value;
+        yaccValue = symbol;
         return result;
     }
 
@@ -745,11 +752,11 @@ private void printToken(int token) {
         switch (token) {
             case RubyParser.yyErrorCode: System.err.print("yyErrorCode,"); break;
             // MISSING tokens
-            case RubyParser.tIDENTIFIER: System.err.print("tIDENTIFIER["+ value() + "],"); break;
-            case RubyParser.tFID: System.err.print("tFID[" + value() + "],"); break;
+            case tIDENTIFIER: System.err.print("tIDENTIFIER["+ value() + "],"); break;
+            case tFID: System.err.print("tFID[" + value() + "],"); break;
             case RubyParser.tGVAR: System.err.print("tGVAR[" + value() + "],"); break;
             case RubyParser.tIVAR: System.err.print("tIVAR[" + value() +"],"); break;
-            case RubyParser.tCONSTANT: System.err.print("tCONSTANT["+ value() +"],"); break;
+            case tCONSTANT: System.err.print("tCONSTANT["+ value() +"],"); break;
             case RubyParser.tCVAR: System.err.print("tCVAR,"); break;
             case RubyParser.tINTEGER: System.err.print("tINTEGER,"); break;
             case RubyParser.tFLOAT: System.err.print("tFLOAT,"); break;
@@ -1078,13 +1085,14 @@ private int yylex() throws IOException {
         }
     }
 
-    private int identifierToken(int result, ByteList value) {
-        if (result == RubyParser.tIDENTIFIER && !isLexState(last_state, EXPR_DOT|EXPR_FNAME) &&
-                parserSupport.getCurrentScope().isDefined(parserSupport.symbol(value)) >= 0) {
+    private int identifierToken(int result, RubySymbol value) {
+        if (result == tIDENTIFIER && !isLexState(last_state, EXPR_DOT|EXPR_FNAME) &&
+                parserSupport.getCurrentScope().isDefined(value) >= 0) {
             setState(EXPR_END|EXPR_LABEL);
         }
 
         yaccValue = value;
+
         return result;
     }
 
@@ -1307,7 +1315,7 @@ private int dollar() throws IOException {
 
                 last_state = lex_state;
                 setState(EXPR_END);
-                yaccValue = createTokenByteList();
+                yaccValue = createTokenSymbol();
                 return RubyParser.tGVAR;
 
             }
@@ -1330,7 +1338,7 @@ private int dollar() throws IOException {
         case '<':       /* $<: reading filename */
         case '>':       /* $>: default output handle */
         case '\"':      /* $": already loaded files */
-            yaccValue = new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING);
+            yaccValue = parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING));
             return RubyParser.tGVAR;
 
 
@@ -1343,7 +1351,7 @@ private int dollar() throws IOException {
                 pushback('-');
                 return '$';
             }
-            yaccValue = createTokenByteList();
+            yaccValue = createTokenSymbol();
             /* xxx shouldn't check if valid option variable */
             return RubyParser.tGVAR;
 
@@ -1353,7 +1361,7 @@ private int dollar() throws IOException {
         case '+':       /* $+: string matches last paren. */
             // Explicit reference to these vars as symbols...
             if (isLexState(last_state, EXPR_FNAME)) {
-                yaccValue = new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING);
+                yaccValue = parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c}, USASCII_ENCODING));
                 return RubyParser.tGVAR;
             }
 
@@ -1367,7 +1375,7 @@ private int dollar() throws IOException {
             } while (Character.isDigit(c));
             pushback(c);
             if (isLexState(last_state, EXPR_FNAME)) {
-                yaccValue = createTokenByteList();
+                yaccValue = createTokenSymbol();
                 return RubyParser.tGVAR;
             }
 
@@ -1386,7 +1394,7 @@ private int dollar() throws IOException {
         case '0':
             setState(EXPR_END);
 
-            return identifierToken(RubyParser.tGVAR, new ByteList(new byte[] {'$', (byte) c}));
+            return identifierToken(RubyParser.tGVAR, parserSupport.symbol(new ByteList(new byte[] {'$', (byte) c})));
         default:
             if (!isIdentifierChar(c)) {
                 if (c == EOF || Character.isSpaceChar(c)) {
@@ -1402,7 +1410,7 @@ private int dollar() throws IOException {
 
             tokadd_ident(c);
 
-            return identifierToken(RubyParser.tGVAR, createTokenByteList());  // $blah
+            return identifierToken(RubyParser.tGVAR, createTokenSymbol());  // $blah
         }
     }
 
@@ -1491,18 +1499,18 @@ private int identifier(int c, boolean commandState) throws IOException {
         int result = 0;
 
         last_state = lex_state;
-        ByteList tempVal;
+        RubySymbol tempVal;
         if (lastBangOrPredicate) {
-            result = RubyParser.tFID;
-            tempVal = createTokenByteList();
+            result = tFID;
+            tempVal = createTokenSymbol();
         } else {
             if (isLexState(lex_state, EXPR_FNAME)) {
                 if ((c = nextc()) == '=') { 
                     int c2 = nextc();
 
                     if (c2 != '~' && c2 != '>' &&
                             (c2 != '=' || peek('>'))) {
-                        result = RubyParser.tIDENTIFIER;
+                        result = tIDENTIFIER;
                         pushback(c2);
                     } else {
                         pushback(c2);
@@ -1512,12 +1520,12 @@ private int identifier(int c, boolean commandState) throws IOException {
                     pushback(c);
                 }
             }
-            tempVal = createTokenByteList();
+            tempVal = createTokenSymbol();
 
-            if (result == 0 && Character.isUpperCase(StringSupport.preciseCodePoint(getEncoding(), tempVal.unsafeBytes(), tempVal.begin(), tempVal.begin() + 1))) {
-                result = RubyParser.tCONSTANT;
+            if (result == 0 && Character.isUpperCase(StringSupport.preciseCodePoint(getEncoding(), tempVal.getBytes().unsafeBytes(), tempVal.getBytes().begin(), tempVal.getBytes().begin() + 1))) {
+                result = tCONSTANT;
             } else {
-                result = RubyParser.tIDENTIFIER;
+                result = tIDENTIFIER;
             }
         }
 
@@ -1531,7 +1539,7 @@ private int identifier(int c, boolean commandState) throws IOException {
         }
 
         if (lex_state != EXPR_DOT) {
-            Keyword keyword = getKeyword(tempVal); // Is it is a keyword?
+            Keyword keyword = getKeyword(tempVal.getBytes()); // Is it is a keyword?
 
             if (keyword != null) {
                 int state = lex_state; // Save state at time keyword is encountered

diff --git a/core/src/main/java/org/jruby/parser/ParserSupport.java b/core/src/main/java/org/jruby/parser/ParserSupport.java
@@ -1113,6 +1113,11 @@ public KeyValuePair<Node, Node> createKeyValue(Node key, Node value) {
         return new KeyValuePair<>(key, value);
     }
 
+    public Node asSymbol(ISourcePosition position, RubySymbol value) {
+        return new SymbolNode(position, value);
+    }
+
+    @Deprecated
     public Node asSymbol(ISourcePosition position, ByteList value) {
         return new SymbolNode(position, symbol(value));
     }

diff --git a/core/src/main/java/org/jruby/parser/RubyParser.java b/core/src/main/java/org/jruby/parser/RubyParser.java
diff --git a/core/src/main/java/org/jruby/parser/RubyParser.y b/core/src/main/java/org/jruby/parser/RubyParser.y
diff --git a/core/src/main/java/org/jruby/parser/YyTables.java b/core/src/main/java/org/jruby/parser/YyTables.java