Skip to content

Commit

Permalink
Fixes #4819. Coverage levels discrepancies between JRuby 1.7.x and JR…
Browse files Browse the repository at this point in the history 
…uby 9.1.x

Basically anything which represents an ident during lexing ivar, cvar, gvar, keywords, constants would ask lexer for current position but lexer will advance past the ident in cases where the ident is last thing on a line and advance to next line (not 100% of the time but most of the time).

Fortunately, the lexer has a special position value for idents called tokline. I switched all obvious nodes made directly from idents to use tokline and it seems to have fixed this issue and probably many others.
@enebo
enebo committed Oct 19, 2017
1 parent 0fd3be0 commit f0b0360
Showing 2 changed files with 57 additions and 57 deletions.
58 changes: 29 additions & 29 deletions core/src/main/java/org/jruby/parser/RubyParser.java
View file
Original file line number Diff line number Diff line change
@@ -2274,27 +2274,27 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[93] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new InstAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new InstAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[94] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new GlobalAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new GlobalAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[95] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

yyVal = new ConstDeclNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
yyVal = new ConstDeclNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
return yyVal;
}
};
states[96] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new ClassVarAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new ClassVarAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
@@ -2389,7 +2389,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
support.yyerror("dynamic constant assignment");
}

ISourcePosition position = lexer.getPosition();
ISourcePosition position = lexer.tokline;

yyVal = new ConstDeclNode(position, (ByteList) null, support.new_colon3(position, ((ByteList)yyVals[0+yyTop])), NilImplicitNode.NIL);
return yyVal;
@@ -2409,27 +2409,27 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[112] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new InstAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new InstAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[113] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new GlobalAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new GlobalAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[114] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

yyVal = new ConstDeclNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
yyVal = new ConstDeclNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
return yyVal;
}
};
states[115] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new ClassVarAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new ClassVarAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
@@ -2524,7 +2524,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
support.yyerror("dynamic constant assignment");
}

ISourcePosition position = lexer.getPosition();
ISourcePosition position = lexer.tokline;

yyVal = new ConstDeclNode(position, (ByteList) null, support.new_colon3(position, ((ByteList)yyVals[0+yyTop])), NilImplicitNode.NIL);
return yyVal;
@@ -2550,13 +2550,13 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[132] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = support.new_colon3(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = support.new_colon3(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
states[133] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = support.new_colon2(lexer.getPosition(), null, ((ByteList)yyVals[0+yyTop]));
yyVal = support.new_colon2(lexer.tokline, null, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
@@ -3649,7 +3649,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[315] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = support.new_colon3(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = support.new_colon3(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
@@ -4886,68 +4886,68 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[512] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new InstVarNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = new InstVarNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
states[513] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new GlobalVarNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = new GlobalVarNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
states[514] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new ConstNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = new ConstNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
states[515] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new ClassVarNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]));
yyVal = new ClassVarNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]));
return yyVal;
}
};
states[516] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new NilNode(lexer.getPosition());
yyVal = new NilNode(lexer.tokline);
return yyVal;
}
};
states[517] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new SelfNode(lexer.getPosition());
yyVal = new SelfNode(lexer.tokline);
return yyVal;
}
};
states[518] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new TrueNode(lexer.getPosition());
yyVal = new TrueNode(lexer.tokline);
return yyVal;
}
};
states[519] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new FalseNode(lexer.getPosition());
yyVal = new FalseNode(lexer.tokline);
return yyVal;
}
};
states[520] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new FileNode(lexer.getPosition(), new ByteList(lexer.getFile().getBytes(),
yyVal = new FileNode(lexer.tokline, new ByteList(lexer.getFile().getBytes(),
support.getConfiguration().getRuntime().getEncodingService().getLocaleEncoding()));
return yyVal;
}
};
states[521] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new FixnumNode(lexer.getPosition(), lexer.tokline.getLine()+1);
yyVal = new FixnumNode(lexer.tokline, lexer.tokline.getLine()+1);
return yyVal;
}
};
states[522] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new EncodingNode(lexer.getPosition(), lexer.getEncoding());
yyVal = new EncodingNode(lexer.tokline, lexer.getEncoding());
return yyVal;
}
};
@@ -4959,27 +4959,27 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
};
states[524] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new InstAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new InstAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[525] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new GlobalAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new GlobalAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
states[526] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

yyVal = new ConstDeclNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
yyVal = new ConstDeclNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), null, NilImplicitNode.NIL);
return yyVal;
}
};
states[527] = new ParserState() {
@Override public Object execute(ParserSupport support, RubyLexer lexer, Object yyVal, Object[] yyVals, int yyTop) {
yyVal = new ClassVarAsgnNode(lexer.getPosition(), ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
yyVal = new ClassVarAsgnNode(lexer.tokline, ((ByteList)yyVals[0+yyTop]), NilImplicitNode.NIL);
return yyVal;
}
};
@@ -5675,7 +5675,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
}
};
}
// line 2732 "RubyParser.y"
// line 2733 "RubyParser.y"

/** The parse method use an lexer stream and parse it to an AST node
* structure
56 changes: 28 additions & 28 deletions core/src/main/java/org/jruby/parser/RubyParser.y
View file
Original file line number Diff line number Diff line change
@@ -689,18 +689,18 @@ mlhs_node : /*mri:user_variable*/ tIDENTIFIER {
$$ = support.assignableLabelOrIdentifier($1, null);
}
| tIVAR {
$$ = new InstAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new InstAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tGVAR {
$$ = new GlobalAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new GlobalAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tCONSTANT {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

$$ = new ConstDeclNode(lexer.getPosition(), $1, null, NilImplicitNode.NIL);
$$ = new ConstDeclNode(lexer.tokline, $1, null, NilImplicitNode.NIL);
}
| tCVAR {
$$ = new ClassVarAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new ClassVarAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
} /*mri:user_variable*/
| /*mri:keyword_variable*/ keyword_nil {
support.compile_error("Can't assign to nil");
@@ -756,7 +756,7 @@ mlhs_node : /*mri:user_variable*/ tIDENTIFIER {
support.yyerror("dynamic constant assignment");
}

ISourcePosition position = lexer.getPosition();
ISourcePosition position = lexer.tokline;

$$ = new ConstDeclNode(position, (ByteList) null, support.new_colon3(position, $2), NilImplicitNode.NIL);
}
@@ -768,18 +768,18 @@ lhs : /*mri:user_variable*/ tIDENTIFIER {
$$ = support.assignableLabelOrIdentifier($1, null);
}
| tIVAR {
$$ = new InstAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new InstAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tGVAR {
$$ = new GlobalAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new GlobalAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tCONSTANT {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

$$ = new ConstDeclNode(lexer.getPosition(), $1, null, NilImplicitNode.NIL);
$$ = new ConstDeclNode(lexer.tokline, $1, null, NilImplicitNode.NIL);
}
| tCVAR {
$$ = new ClassVarAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new ClassVarAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
} /*mri:user_variable*/
| /*mri:keyword_variable*/ keyword_nil {
support.compile_error("Can't assign to nil");
@@ -835,7 +835,7 @@ lhs : /*mri:user_variable*/ tIDENTIFIER {
support.yyerror("dynamic constant assignment");
}

ISourcePosition position = lexer.getPosition();
ISourcePosition position = lexer.tokline;

$$ = new ConstDeclNode(position, (ByteList) null, support.new_colon3(position, $2), NilImplicitNode.NIL);
}
@@ -851,10 +851,10 @@ cname : tIDENTIFIER {
}

cpath : tCOLON3 cname {
$$ = support.new_colon3(lexer.getPosition(), $2);
$$ = support.new_colon3(lexer.tokline, $2);
}
| cname {
$$ = support.new_colon2(lexer.getPosition(), null, $1);
$$ = support.new_colon2(lexer.tokline, null, $1);
}
| primary_value tCOLON2 cname {
$$ = support.new_colon2(support.getPosition($1), $1, $3);
@@ -1481,7 +1481,7 @@ primary : literal
$$ = support.new_colon2(support.getPosition($1), $1, $3);
}
| tCOLON3 tCONSTANT {
$$ = support.new_colon3(lexer.getPosition(), $2);
$$ = support.new_colon3(lexer.tokline, $2);
}
| tLBRACK aref_args tRBRACK {
ISourcePosition position = support.getPosition($2);
@@ -2241,57 +2241,57 @@ var_ref : /*mri:user_variable*/ tIDENTIFIER {
$$ = support.declareIdentifier($1);
}
| tIVAR {
$$ = new InstVarNode(lexer.getPosition(), $1);
$$ = new InstVarNode(lexer.tokline, $1);
}
| tGVAR {
$$ = new GlobalVarNode(lexer.getPosition(), $1);
$$ = new GlobalVarNode(lexer.tokline, $1);
}
| tCONSTANT {
$$ = new ConstNode(lexer.getPosition(), $1);
$$ = new ConstNode(lexer.tokline, $1);
}
| tCVAR {
$$ = new ClassVarNode(lexer.getPosition(), $1);
$$ = new ClassVarNode(lexer.tokline, $1);
} /*mri:user_variable*/
| /*mri:keyword_variable*/ keyword_nil {
$$ = new NilNode(lexer.getPosition());
$$ = new NilNode(lexer.tokline);
}
| keyword_self {
$$ = new SelfNode(lexer.getPosition());
$$ = new SelfNode(lexer.tokline);
}
| keyword_true {
$$ = new TrueNode(lexer.getPosition());
$$ = new TrueNode(lexer.tokline);
}
| keyword_false {
$$ = new FalseNode(lexer.getPosition());
$$ = new FalseNode(lexer.tokline);
}
| keyword__FILE__ {
$$ = new FileNode(lexer.getPosition(), new ByteList(lexer.getFile().getBytes(),
$$ = new FileNode(lexer.tokline, new ByteList(lexer.getFile().getBytes(),
support.getConfiguration().getRuntime().getEncodingService().getLocaleEncoding()));
}
| keyword__LINE__ {
$$ = new FixnumNode(lexer.getPosition(), lexer.tokline.getLine()+1);
$$ = new FixnumNode(lexer.tokline, lexer.tokline.getLine()+1);
}
| keyword__ENCODING__ {
$$ = new EncodingNode(lexer.getPosition(), lexer.getEncoding());
$$ = new EncodingNode(lexer.tokline, lexer.getEncoding());
} /*mri:keyword_variable*/

// [!null]
var_lhs : /*mri:user_variable*/ tIDENTIFIER {
$$ = support.assignableLabelOrIdentifier($1, null);
}
| tIVAR {
$$ = new InstAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new InstAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tGVAR {
$$ = new GlobalAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new GlobalAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
}
| tCONSTANT {
if (support.isInDef() || support.isInSingle()) support.compile_error("dynamic constant assignment");

$$ = new ConstDeclNode(lexer.getPosition(), $1, null, NilImplicitNode.NIL);
$$ = new ConstDeclNode(lexer.tokline, $1, null, NilImplicitNode.NIL);
}
| tCVAR {
$$ = new ClassVarAsgnNode(lexer.getPosition(), $1, NilImplicitNode.NIL);
$$ = new ClassVarAsgnNode(lexer.tokline, $1, NilImplicitNode.NIL);
} /*mri:user_variable*/
| /*mri:keyword_variable*/ keyword_nil {
support.compile_error("Can't assign to nil");

0 comments on commit f0b0360

Please sign in to comment.