Fixes #2154. Parser needs to support "foo": hash key syntax

jruby · Jan 18, 2015 · 39c7e33 · 39c7e33
1 parent a1e5645
commit 39c7e33
Showing 9 changed files with 4,038 additions and 3,909 deletions.
diff --git a/core/src/main/java/org/jruby/lexer/yacc/HeredocTerm.java b/core/src/main/java/org/jruby/lexer/yacc/HeredocTerm.java
@@ -65,10 +65,6 @@ public HeredocTerm(ByteList marker, int func, ByteList lastLine) {
         this.lastLine = lastLine;
     }
 
-    public int getNest() {
-        return 0;
-    }
-
     public int parseString(RubyLexer lexer, LexerSource src) throws java.io.IOException {
         boolean indent = (flags & RubyLexer.STR_FUNC_INDENT) != 0;
 

diff --git a/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java b/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
@@ -326,6 +326,7 @@ public void newtok() {
 
     // Count of nested parentheses
     private int parenNest = 0;
+    private int braceNest = 0;
 
     private int leftParenBegin = 0;
 
@@ -335,6 +336,14 @@ public int incrementParenNest() {
         return parenNest;
     }
 
+    public int getBraceNest() {
+        return braceNest;
+    }
+
+    public void setBraceNest(int nest) {
+        braceNest = nest;
+    }
+
     public int getLeftParenBegin() {
         return leftParenBegin;
     }
@@ -356,6 +365,8 @@ public final void reset() {
         resetStacks();
         lex_strterm = null;
         commandStart = true;
+        parenNest = 0;
+        braceNest = 0;
     }
 
     public int nextToken() throws IOException {
@@ -469,6 +480,10 @@ private void printState() {
         }
     }
 
+    public LexState getState() {
+        return lex_state;
+    }
+
     public void setState(LexState state) {
         this.lex_state = state;
 //        printState();
@@ -1094,7 +1109,16 @@ private int yylex() throws IOException {
 
         if (lex_strterm != null) {
             int tok = lex_strterm.parseString(this, src);
-            if (tok == Tokens.tSTRING_END || tok == Tokens.tREGEXP_END) {
+
+            if (tok == Tokens.tSTRING_END && (yaccValue.equals("\"") || yaccValue.equals("'"))) {
+                if (((lex_state == LexState.EXPR_BEG || lex_state == LexState.EXPR_ENDFN) && !conditionState.isInState() ||
+                        isARG()) && src.peek(':')) {
+                    src.read();
+                    tok = Tokens.tLABEL_END;
+                }
+            }
+
+            if (tok == Tokens.tSTRING_END || tok == Tokens.tREGEXP_END || tok == Tokens.tLABEL_END) {
                 lex_strterm = null;
                 setState(LexState.EXPR_END);
             }
@@ -1861,6 +1885,8 @@ private int leftBracket(boolean spaceSeen) throws IOException {
     }
 
     private int leftCurly() {
+        braceNest++;
+        //System.out.println("lcurly: " + braceNest);
         if (leftParenBegin > 0 && leftParenBegin == parenNest) {
             setState(LexState.EXPR_BEG);
             leftParenBegin = 0;
@@ -2182,7 +2208,10 @@ private int rightCurly() {
         cmdArgumentState.restart();
         setState(LexState.EXPR_ENDARG);
         yaccValue = "}";
-        return Tokens.tRCURLY;
+        //System.out.println("braceNest: " + braceNest);
+        int tok = /*braceNest != 0 ? Tokens.tSTRING_DEND : */ Tokens.tRCURLY;
+        braceNest--;
+        return tok;
     }
 
     private int rightParen() {

diff --git a/core/src/main/java/org/jruby/lexer/yacc/StrTerm.java b/core/src/main/java/org/jruby/lexer/yacc/StrTerm.java
@@ -28,6 +28,5 @@
 package org.jruby.lexer.yacc;
 
 public abstract class StrTerm {
-    public abstract int getNest();
     public abstract int parseString(RubyLexer lexer, LexerSource src) throws java.io.IOException;
 }
diff --git a/core/src/main/java/org/jruby/lexer/yacc/StringTerm.java b/core/src/main/java/org/jruby/lexer/yacc/StringTerm.java
@@ -56,10 +56,6 @@ public StringTerm(int flags, int begin, int end) {
         this.nest  = 0;
     }
 
-    public int getNest() {
-        return nest;
-    }
-
     protected ByteList createByteList(RubyLexer lexer) {
         return new ByteList(new byte[]{}, lexer.getEncoding());
     }
@@ -134,6 +130,7 @@ private int parsePeekVariableName(RubyLexer lexer, LexerSource src) throws IOExc
                 break;
             }
             case '{':
+                //lexer.setBraceNest(lexer.getBraceNest() + 1);
                 lexer.setValue("#" + (char) c);
                 return Tokens.tSTRING_DBEG;
             default:

diff --git a/core/src/main/java/org/jruby/parser/RubyParser.java b/core/src/main/java/org/jruby/parser/RubyParser.java
diff --git a/core/src/main/java/org/jruby/parser/RubyParser.y b/core/src/main/java/org/jruby/parser/RubyParser.y
@@ -213,7 +213,8 @@ public class RubyParser {
 %token <RegexpNode>  tREGEXP_END
 %type <RestArgNode> f_rest_arg 
 %type <Node> singleton strings string string1 xstring regexp
-%type <Node> string_contents xstring_contents string_content method_call
+%type <Node> string_contents xstring_contents method_call
+%type <Object> string_content
 %type <Node> regexp_contents
 %type <Node> words qwords word literal numeric simple_numeric dsym cpath command_asgn command_call
 %type <Node> mrhs_arg
@@ -264,6 +265,7 @@ public class RubyParser {
 %token <String> tSTRING_DEND
 %type <String> kwrest_mark, f_kwrest, f_label
 %type <FCallNode> fcall
+%token <String> tLABEL_END, tSTRING_DEND
 
 /*
  *    precedence table
@@ -1909,9 +1911,11 @@ word_list       : /* none */ {
                      $$ = $1.add($2 instanceof EvStrNode ? new DStrNode($1.getPosition(), lexer.getEncoding()).add($2) : $2);
                 }
 
-word            : string_content
+word            : string_content {
+                     $$ = $<Node>1;
+                }
                 | word string_content {
-                     $$ = support.literal_concat(support.getPosition($1), $1, $2);
+                     $$ = support.literal_concat(support.getPosition($1), $1, $<Node>2);
                 }
 
 symbols         : tSYMBOLS_BEG ' ' tSTRING_END {
@@ -1963,22 +1967,22 @@ string_contents : /* none */ {
                     $$ = lexer.createStrNode(lexer.getPosition(), aChar, 0);
                 }
                 | string_contents string_content {
-                    $$ = support.literal_concat($1.getPosition(), $1, $2);
+                    $$ = support.literal_concat($1.getPosition(), $1, $<Node>2);
                 }
 
 xstring_contents: /* none */ {
                     $$ = null;
                 }
                 | xstring_contents string_content {
-                    $$ = support.literal_concat(support.getPosition($1), $1, $2);
+                    $$ = support.literal_concat(support.getPosition($1), $1, $<Node>2);
                 }
 
 regexp_contents :  /* none */ {
                     $$ = null;
                 }
                 | regexp_contents string_content {
     // FIXME: mri is different here.
-                    $$ = support.literal_concat(support.getPosition($1), $1, $2);
+                    $$ = support.literal_concat(support.getPosition($1), $1, $<Node>2);
                 }
 
 string_content  : tSTRING_CONTENT {
@@ -1994,16 +1998,19 @@ string_content  : tSTRING_CONTENT {
                 }
                 | tSTRING_DBEG {
                    $$ = lexer.getStrTerm();
+                   lexer.setStrTerm(null);
                    lexer.getConditionState().stop();
                    lexer.getCmdArgumentState().stop();
-                   lexer.setStrTerm(null);
+                } {
+                   $$ = lexer.getState();
                    lexer.setState(LexState.EXPR_BEG);
                 } compstmt tRCURLY {
                    lexer.getConditionState().restart();
                    lexer.getCmdArgumentState().restart();
                    lexer.setStrTerm($<StrTerm>2);
+                   lexer.setState($<LexState>3);
 
-                   $$ = support.newEvStrNode(support.getPosition($3), $3);
+                   $$ = support.newEvStrNode(support.getPosition($4), $4);
                 }
 
 string_dvar     : tGVAR {
@@ -2446,6 +2453,19 @@ assoc           : arg_value tASSOC arg_value {
                     SymbolNode label = new SymbolNode(support.getPosition($2), new ByteList($1.getBytes(), lexer.getEncoding()));
                     $$ = new KeyValuePair<Node,Node>(label, $2);
                 }
+                | tSTRING_BEG string_contents tLABEL_END arg_value {
+                    if ($2 instanceof StrNode) {
+                        DStrNode dnode = new DStrNode(support.getPosition($2), lexer.getEncoding());
+                        dnode.add($2);
+                        $$ = new KeyValuePair<Node,Node>(new DSymbolNode(support.getPosition($2), dnode), $4);
+                    } else if ($2 instanceof DStrNode) {
+                        $$ = new KeyValuePair<Node,Node>(new DSymbolNode(support.getPosition($2), $<DStrNode>2), $4);
+                    } else {
+                        support.compile_error("Uknown type for assoc in strings: " + $2);
+                    }
+
+                }
+
                 | tDSTAR arg_value {
                     $$ = new KeyValuePair<Node,Node>(null, $2);
                 }

diff --git a/core/src/main/java/org/jruby/parser/Tokens.java b/core/src/main/java/org/jruby/parser/Tokens.java
@@ -161,6 +161,8 @@ public interface Tokens {
     int tSYMBOLS_BEG = RubyParser.tSYMBOLS_BEG;
     int tQSYMBOLS_BEG = RubyParser.tQSYMBOLS_BEG;
     int tDSTAR = RubyParser.tDSTAR;
+    int tLABEL_END = RubyParser.tLABEL_END;
+    int tSTRING_DEND = RubyParser.tSTRING_DEND;
 
     String[] operators = {"+@", "-@", "**", "<=>", "==", "===", "!=", ">=", "<=", "&&",
                           "||", "=~", "!~", "..", "...", "[]", "[]=", "<<", ">>", "::"};

diff --git a/core/src/main/java/org/jruby/parser/YyTables.java b/core/src/main/java/org/jruby/parser/YyTables.java
diff --git a/test/mri/excludes/TestHash.rb b/test/mri/excludes/TestHash.rb
@@ -10,11 +10,10 @@
 exclude :test_hash_bignum_hash, "needs investigation"
 exclude :test_hash2, "[ruby-core:58993] [Bug #9231]"
 exclude :test_inverse_hash, "needs investigation"
-exclude :test_label_syntax, "waiting on parser support (#2154)"
 exclude :test_NEWHASH_fstring_key, "needs investigation"
 exclude :test_replace2, "needs investigation"
 exclude :test_recursive_hash_value_struct, "needs investigation"
 exclude :test_reject, "needs investigation"
 exclude :test_shift_none, "needs investigation"
 exclude :test_to_a, "needs investigation"
-exclude :test_AREF_fstring_key, "Depends on MRI-specific GC.stat key"
+exclude :test_AREF_fstring_key, "Depends on MRI-specific GC.stat key"