Skip to content

Commit

Permalink
Merge remote-tracking branch 'github/master' into prepend
Browse files Browse the repository at this point in the history
  • Loading branch information
@headius
headius committed Feb 26, 2013
2 parents d4f1931 + 0001ac3 commit f1ff8dd
Show file tree
Hide file tree
Showing 10 changed files with 570 additions and 499 deletions.
2 changes: 1 addition & 1 deletion spec/regression/JRUBY-5122_nonblocking_io_spec.rb
View file
Expand Up @@ -214,7 +214,7 @@
# Oracle's build block with > 131072 (2**17)
# On a Windows 7(64) box:
# Oracle's build does not block (use memory till OOMException)
SOCKET_CHANNEL_MIGHT_BLOCK = "a" * (65536 * 4)
SOCKET_CHANNEL_MIGHT_BLOCK = "a" * (219463 * 4)

it "should not block for write" do
100.times do # for acceleration; it failed w/o wait_for_accepted call
Expand Down
61 changes: 40 additions & 21 deletions src/org/jruby/ext/ripper/RipperLexer.java
View file
Expand Up @@ -433,6 +433,10 @@ private boolean isEND() {
private boolean isARG() {
return lex_state == LexState.EXPR_ARG || lex_state == LexState.EXPR_CMDARG;
}

private boolean isSpaceArg(int c, boolean spaceSeen) {
return isARG() && spaceSeen && !Character.isWhitespace(c);
}

private void determineExpressionState() {
switch (lex_state) {
Expand Down Expand Up @@ -1121,6 +1125,10 @@ private int yylex() throws IOException {
boolean spaceSeen = false;
boolean commandState;

// FIXME: Sucks we do this n times versus one since it is only important at beginning of parse but we need to change
// setup of parser differently.
if (token == 0 && src.getLine() == 0) detectUTF8BOM();

if (lex_strterm != null) {
int tok = lex_strterm.parseString(this, src);
if (tok == Tokens.tSTRING_END || tok == Tokens.tREGEXP_END) {
Expand Down Expand Up @@ -1413,7 +1421,7 @@ private int ampersand(boolean spaceSeen) throws IOException {
//if the warning is generated, the getPosition() on line 954 (this line + 18) will create
//a wrong position if the "inclusive" flag is not set.
ISourcePosition tmpPosition = getPosition();
if (isARG() && spaceSeen && !Character.isWhitespace(c)) {
if (isSpaceArg(c, spaceSeen)) {
IRubyWarnings warnings = getRuntime().getWarnings();
if (warnings.isVerbose()) warnings.warning(IRubyWarnings.ID.ARGUMENT_AS_PREFIX, tmpPosition, "`&' interpreted as argument prefix");
c = Tokens.tAMPER;
Expand Down Expand Up @@ -1953,7 +1961,7 @@ private int minus(boolean spaceSeen) throws IOException {
setState(LexState.EXPR_ARG);
return Tokens.tLAMBDA;
}
if (isBEG() || (isARG() && spaceSeen && !Character.isWhitespace(c))) {
if (isBEG() || isSpaceArg(c, spaceSeen)) {
if (isARG()) arg_ambiguous();
setState(LexState.EXPR_BEG);
src.unread(c);
Expand All @@ -1978,7 +1986,7 @@ private int percent(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
}

if (isARG() && spaceSeen && !Character.isWhitespace(c)) return parseQuote(c);
if (isSpaceArg(c, spaceSeen)) return parseQuote(c);

determineExpressionState();

Expand Down Expand Up @@ -2026,7 +2034,7 @@ private int plus(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
}

if (isBEG() || (isARG() && spaceSeen && !Character.isWhitespace(c))) {
if (isBEG() || isSpaceArg(c, spaceSeen)) {
if (isARG()) arg_ambiguous();
setState(LexState.EXPR_BEG);
src.unread(c);
Expand Down Expand Up @@ -2161,12 +2169,10 @@ private int slash(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
}
src.unread(c);
if (isARG() && spaceSeen) {
if (!Character.isWhitespace(c)) {
arg_ambiguous();
lex_strterm = new StringTerm(str_regexp, '\0', '/');
return Tokens.tREGEXP_BEG;
}
if (isSpaceArg(c, spaceSeen)) {
arg_ambiguous();
lex_strterm = new StringTerm(str_regexp, '\0', '/');
return Tokens.tREGEXP_BEG;
}

determineExpressionState();
Expand All @@ -2191,7 +2197,7 @@ private int star(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
default:
src.unread(c);
if (isARG() && spaceSeen && !Character.isWhitespace(c)) {
if (isSpaceArg(c, spaceSeen)) {
IRubyWarnings warnings = getRuntime().getWarnings();

if (warnings.isVerbose()) warnings.warning(IRubyWarnings.ID.ARGUMENT_AS_PREFIX, getPosition(), "`*' interpreted as argument prefix");
Expand Down Expand Up @@ -2535,11 +2541,6 @@ public int readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolL
buffer.setEncoding(UTF8_ENCODING);
if (stringLiteral) tokenAddMBC(codepoint, buffer);
} else if (stringLiteral) {
if (codepoint == 0 && symbolLiteral) {
throw new SyntaxException(SyntaxException.PID.INVALID_ESCAPE_SYNTAX, getPosition(),
getCurrentLine(), "symbol cannot contain '\\u0000'");
}

buffer.append((char) codepoint);
}
} while (src.peek(' ') || src.peek('\t'));
Expand All @@ -2555,11 +2556,6 @@ public int readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolL
buffer.setEncoding(UTF8_ENCODING);
if (stringLiteral) tokenAddMBC(codepoint, buffer);
} else if (stringLiteral) {
if (codepoint == 0 && symbolLiteral) {
throw new SyntaxException(SyntaxException.PID.INVALID_ESCAPE_SYNTAX, getPosition(),
getCurrentLine(), "symbol cannot contain '\\u0000'");
}

buffer.append((char) codepoint);
}
}
Expand Down Expand Up @@ -2711,4 +2707,27 @@ private char scanOct(int count) throws IOException {

return value;
}

// FIXME: Also sucks that matchMarker will strip off valuable bytes and not work for this (could be a one-liner)
private void detectUTF8BOM() throws IOException {
int b1 = src.read();
if (b1 == 0xef) {
int b2 = src.read();
if (b2 == 0xbb) {
int b3 = src.read();
if (b3 == 0xbf) {
setEncoding(UTF8_ENCODING);
} else {
src.unread(b3);
src.unread(b2);
src.unread(b1);
}
} else {
src.unread(b2);
src.unread(b1);
}
} else {
src.unread(b1);
}
}
}
2 changes: 0 additions & 2 deletions src/org/jruby/internal/runtime/methods/DefaultMethod.java
View file
Expand Up @@ -142,8 +142,6 @@ private DynamicMethod tryJitReturnMethod(ThreadContext context) {
// use the class name
className = implementationClass.getName();
}
// replace double-colons with dots, to match Java
className.replaceAll("::", ".");
context.runtime.getJITCompiler().tryJIT(this, context, className, name);
return box.actualMethod;
}
Expand Down
105 changes: 76 additions & 29 deletions src/org/jruby/lexer/yacc/RubyYaccLexer.java
View file
Expand Up @@ -159,6 +159,29 @@ private void warn_balanced(int c, boolean spaceSeen, String op, String syn) {
ambiguousOperator(op, syn);
}
}

// FIXME: Also sucks that matchMarker will strip off valuable bytes and not work for this (could be a one-liner)
private void detectUTF8BOM() throws IOException {
int b1 = src.read();
if (b1 == 0xef) {
int b2 = src.read();
if (b2 == 0xbb) {
int b3 = src.read();
if (b3 == 0xbf) {
setEncoding(UTF8_ENCODING);
} else {
src.unread(b3);
src.unread(b2);
src.unread(b1);
}
} else {
src.unread(b2);
src.unread(b1);
}
} else {
src.unread(b1);
}
}

public enum Keyword {
END ("end", Tokens.kEND, Tokens.kEND, LexState.EXPR_END),
Expand Down Expand Up @@ -275,6 +298,7 @@ public static Keyword getKeyword(String str) {

// Are we lexing Ruby 1.8 or 1.9+ syntax
private boolean isOneEight;
private boolean isTwoZero;
// Count of nested parentheses (1.9 only)
private int parenNest = 0;
// 1.9 only
Expand Down Expand Up @@ -311,6 +335,7 @@ public final void reset() {
resetStacks();
lex_strterm = null;
commandStart = true;
if (parserSupport != null) isTwoZero = parserSupport.getConfiguration().getVersion().is2_0();
}

public int nextToken() throws IOException {
Expand Down Expand Up @@ -467,6 +492,10 @@ private boolean isEND() {
private boolean isARG() {
return lex_state == LexState.EXPR_ARG || lex_state == LexState.EXPR_CMDARG;
}

private boolean isSpaceArg(int c, boolean spaceSeen) {
return isARG() && spaceSeen && !Character.isWhitespace(c);
}

private void determineExpressionState() {
switch (lex_state) {
Expand Down Expand Up @@ -621,10 +650,26 @@ private int parseQuote(int c) throws IOException {
setState(LexState.EXPR_FNAME);
yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tSYMBEG;


case 'I':
if (isTwoZero) {
lex_strterm = new StringTerm(str_dquote | STR_FUNC_QWORDS, begin, end);
do {c = src.read();} while (Character.isWhitespace(c));
src.unread(c);
yaccValue = new Token("%" + c + begin, getPosition());
return Tokens.tSYMBOLS_BEG;
}
case 'i':
if (isTwoZero) {
lex_strterm = new StringTerm(/* str_squote | */STR_FUNC_QWORDS, begin, end);
do {c = src.read();} while (Character.isWhitespace(c));
src.unread(c);
yaccValue = new Token("%" + c + begin, getPosition());
return Tokens.tQSYMBOLS_BEG;
}
default:
throw new SyntaxException(PID.STRING_UNKNOWN_TYPE, getPosition(), getCurrentLine(),
"Unknown type of %string. Expected 'Q', 'q', 'w', 'x', 'r' or any non letter character, but found '" + c + "'.");
throw new SyntaxException(PID.STRING_UNKNOWN_TYPE,
getPosition(), getCurrentLine(), "unknown type of %string");
}
}

Expand Down Expand Up @@ -966,7 +1011,7 @@ private int yylex2() throws IOException {

return currentToken;
}

/**
* Returns the next token. Also sets yyVal is needed.
*
Expand All @@ -976,6 +1021,10 @@ private int yylex() throws IOException {
int c;
boolean spaceSeen = false;
boolean commandState;

// FIXME: Sucks we do this n times versus one since it is only important at beginning of parse but we need to change
// setup of parser differently.
if (token == 0 && src.getLine() == 0) detectUTF8BOM();

if (lex_strterm != null) {
int tok = lex_strterm.parseString(this, src);
Expand Down Expand Up @@ -1255,7 +1304,7 @@ private int ampersand(boolean spaceSeen) throws IOException {
//if the warning is generated, the getPosition() on line 954 (this line + 18) will create
//a wrong position if the "inclusive" flag is not set.
ISourcePosition tmpPosition = getPosition();
if (isARG() && spaceSeen && !Character.isWhitespace(c)) {
if (isSpaceArg(c, spaceSeen)) {
if (warnings.isVerbose()) warnings.warning(ID.ARGUMENT_AS_PREFIX, tmpPosition, "`&' interpreted as argument prefix");
c = Tokens.tAMPER;
} else if (isBEG()) {
Expand Down Expand Up @@ -1854,7 +1903,7 @@ private int minus(boolean spaceSeen) throws IOException {
yaccValue = new Token("->", getPosition());
return Tokens.tLAMBDA;
}
if (isBEG() || (isARG() && spaceSeen && !Character.isWhitespace(c))) {
if (isBEG() || isSpaceArg(c, spaceSeen)) {
if (isARG()) arg_ambiguous();
setState(LexState.EXPR_BEG);
src.unread(c);
Expand All @@ -1881,8 +1930,8 @@ private int percent(boolean spaceSeen) throws IOException {
yaccValue = new Token("%", getPosition());
return Tokens.tOP_ASGN;
}
if (isARG() && spaceSeen && !Character.isWhitespace(c)) return parseQuote(c);

if (isSpaceArg(c, spaceSeen)) return parseQuote(c);

determineExpressionState();

Expand Down Expand Up @@ -1938,7 +1987,7 @@ private int plus(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
}

if (isBEG() || (isARG() && spaceSeen && !Character.isWhitespace(c))) {
if (isBEG() || isSpaceArg(c, spaceSeen)) { //FIXME: arg_ambiguous missing
if (isARG()) arg_ambiguous();
setState(LexState.EXPR_BEG);
src.unread(c);
Expand Down Expand Up @@ -2084,13 +2133,11 @@ private int slash(boolean spaceSeen) throws IOException {
return Tokens.tOP_ASGN;
}
src.unread(c);
if (isARG() && spaceSeen) {
if (!Character.isWhitespace(c)) {
arg_ambiguous();
lex_strterm = new StringTerm(str_regexp, '\0', '/');
yaccValue = new Token("/",getPosition());
return Tokens.tREGEXP_BEG;
}
if (isSpaceArg(c, spaceSeen)) {
arg_ambiguous();
lex_strterm = new StringTerm(str_regexp, '\0', '/');
yaccValue = new Token("/",getPosition());
return Tokens.tREGEXP_BEG;
}

determineExpressionState();
Expand All @@ -2110,17 +2157,27 @@ private int star(boolean spaceSeen) throws IOException {
yaccValue = new Token("**", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);

src.unread(c); // not a '=' put it back
yaccValue = new Token("**", getPosition());
c = Tokens.tPOW;

if (isTwoZero && isSpaceArg(c, spaceSeen)) {
if (warnings.isVerbose()) warnings.warning(ID.ARGUMENT_AS_PREFIX, getPosition(), "`**' interpreted as argument prefix");
c = Tokens.tDSTAR;
} else if (isTwoZero && isBEG()) {
c = Tokens.tDSTAR;
} else {
if (!isOneEight) warn_balanced(c, spaceSeen, "*", "argument prefix");
c = Tokens.tPOW;
}
break;
case '=':
setState(LexState.EXPR_BEG);
yaccValue = new Token("*", getPosition());
return Tokens.tOP_ASGN;
default:
src.unread(c);
if (isARG() && spaceSeen && !Character.isWhitespace(c)) {
if (isSpaceArg(c, spaceSeen)) {
if (warnings.isVerbose()) warnings.warning(ID.ARGUMENT_AS_PREFIX, getPosition(), "`*' interpreted as argument prefix");
c = Tokens.tSTAR;
} else if (isBEG()) {
Expand Down Expand Up @@ -2463,11 +2520,6 @@ public int readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolL
buffer.setEncoding(UTF8_ENCODING);
if (stringLiteral) tokenAddMBC(codepoint, buffer);
} else if (stringLiteral) {
if (codepoint == 0 && symbolLiteral) {
throw new SyntaxException(PID.INVALID_ESCAPE_SYNTAX, getPosition(),
getCurrentLine(), "symbol cannot contain '\\u0000'");
}

buffer.append((char) codepoint);
}
} while (src.peek(' ') || src.peek('\t'));
Expand All @@ -2483,11 +2535,6 @@ public int readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolL
buffer.setEncoding(UTF8_ENCODING);
if (stringLiteral) tokenAddMBC(codepoint, buffer);
} else if (stringLiteral) {
if (codepoint == 0 && symbolLiteral) {
throw new SyntaxException(PID.INVALID_ESCAPE_SYNTAX, getPosition(),
getCurrentLine(), "symbol cannot contain '\\u0000'");
}

buffer.append((char) codepoint);
}
}
Expand Down
1 change: 0 additions & 1 deletion src/org/jruby/parser/.#Ruby19Parser.y
View file

This file was deleted.

7 changes: 7 additions & 0 deletions src/org/jruby/parser/ParserSupport.java
View file
Expand Up @@ -1145,6 +1145,13 @@ public DStrNode createDStrNode(ISourcePosition position) {
return new DStrNode(position);
}

public Node asSymbol(ISourcePosition position, Node value) {
// FIXME: This might have an encoding issue since toString generally uses iso-8859-1
if (value instanceof StrNode) return new SymbolNode(position, ((StrNode) value).getValue().toString());

return new DSymbolNode(position, (DStrNode) value);
}

public Node literal_concat(ISourcePosition position, Node head, Node tail) {
if (head == null) return tail;
if (tail == null) return head;
Expand Down

0 comments on commit f1ff8dd

Please sign in to comment.