Skip to content

Commit 1443fd6

Browse files
eneboenebo
committedMar 26, 2018
Fix issues with string processing of word types not putting exact space
characters for on_space. Surprisingly icky :)
·
10.0.0.09.2.0.0
1 parent 841ae3c commit 1443fd6

File tree

2 files changed

+70
-56
lines changed

2 files changed

+70
-56
lines changed
 

Diff for: ‎core/src/main/java/org/jruby/ext/ripper/RipperLexer.java

+67-54
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,6 @@ public class RipperLexer extends LexingCommon {
9696
map.put("__ENCODING__", Keyword.__ENCODING__);
9797
}
9898

99-
public boolean ignoreNextScanEvent = false;
100-
10199
protected void ambiguousOperator(String op, String syn) {
102100
parser.dispatch("on_operator_ambiguous", getRuntime().newSymbol(op), getRuntime().newString(syn));
103101
}
@@ -248,6 +246,7 @@ public RipperLexer(RipperParserBase parser, LexerSource src) {
248246
protected ByteList delayed = null;
249247
private int delayed_line = 0;
250248
private int delayed_col = 0;
249+
private boolean cr_seen = false;
251250

252251
/**
253252
* Has lexing started yet?
@@ -267,64 +266,76 @@ protected void flush_string_content(Encoding encoding) {
267266
tokp = lex_p;
268267
}
269268
}
270-
271-
public int nextc() {
272-
if (lex_p == lex_pend) {
273-
line_offset += lex_pend;
274269

275-
ByteList v = lex_nextline;
276-
lex_nextline = null;
277-
278-
if (v == null) {
279-
if (eofp) return EOF;
280-
281-
if (src == null || (v = src.gets()) == null) {
282-
eofp = true;
283-
lex_goto_eol();
284-
return EOF;
285-
}
286-
}
287-
288-
// Left over stuffs...Add to delayed for later processing.
289-
if (tokp < lex_pend) {
290-
if (delayed == null) {
291-
delayed = new ByteList();
292-
delayed.setEncoding(getEncoding());
293-
delayed.append(lexb, tokp, lex_pend - tokp);
294-
delayed_line = ruby_sourceline;
295-
delayed_col = tokp - lex_pbeg;
296-
} else {
297-
delayed.append(lexb, tokp, lex_pend - tokp);
298-
}
270+
public void addDelayedToken(int tok, int end) {
271+
// Left over stuffs...Add to delayed for later processing.
272+
if (tok < end) {
273+
if (delayed == null) {
274+
delayed = new ByteList();
275+
delayed.setEncoding(getEncoding());
276+
delayed_line = ruby_sourceline;
277+
delayed_col = tok - lex_pbeg;
299278
}
300-
301-
if (heredoc_end > 0) {
302-
ruby_sourceline = heredoc_end;
303-
heredoc_end = 0;
279+
delayed.append(lexb, tok, end - tok);
280+
tokp = end;
281+
}
282+
}
283+
284+
private boolean nextLine() {
285+
line_offset += lex_pend;
286+
287+
ByteList v = lex_nextline;
288+
lex_nextline = null;
289+
290+
if (v == null) {
291+
if (eofp) return true;
292+
293+
if (src == null || (v = src.gets()) == null) {
294+
eofp = true;
295+
lex_goto_eol();
296+
return true;
304297
}
305-
ruby_sourceline++;
306-
line_count++;
307-
lex_pbeg = lex_p = 0;
308-
lex_pend = lex_p + v.length();
309-
lexb = v;
310-
flush();
311-
lex_lastline = v;
298+
cr_seen = false;
312299
}
313-
300+
301+
addDelayedToken(tokp, lex_pend);
302+
303+
if (heredoc_end > 0) {
304+
ruby_sourceline = heredoc_end;
305+
heredoc_end = 0;
306+
}
307+
ruby_sourceline++;
308+
line_count++;
309+
lex_pbeg = lex_p = 0;
310+
lex_pend = lex_p + v.length();
311+
lexb = v;
312+
flush();
313+
lex_lastline = v;
314+
315+
return false;
316+
}
317+
318+
private int cr(int c) {
319+
if (peek('\n')) {
320+
lex_p++;
321+
c = '\n';
322+
} else if (!cr_seen) {
323+
cr_seen = true;
324+
warn("encountered \\\\r in middle of line, treated as a mere space");
325+
}
326+
return c;
327+
}
328+
329+
public int nextc() {
330+
if (lex_p == lex_pend || eofp || lex_nextline != null) {
331+
if (nextLine()) return EOF;
332+
}
333+
314334
int c = p(lex_p);
315335
lex_p++;
316-
if (c == '\r') {
317-
if (peek('\n')) {
318-
lex_p++;
319-
c = '\n';
320-
} else if (ruby_sourceline > last_cr_line) {
321-
last_cr_line = ruby_sourceline;
322-
warn("encountered \\\\r in middle of line, treated as a mere space");
323-
c = ' ';
324-
}
325-
}
326336

327-
// System.out.println("C: " + (char) c + ", LEXP: " + lex_p + ", PEND: "+ lex_pend);
337+
if (c == '\r') c = cr(c);
338+
328339
return c;
329340
}
330341

@@ -714,7 +725,9 @@ private void printToken(int token) {
714725
}
715726

716727
public boolean hasScanEvent() {
717-
if (lex_p < tokp) throw parser.getRuntime().newRuntimeError("lex_p < tokp");
728+
if (lex_p < tokp) {
729+
throw parser.getRuntime().newRuntimeError("lex_p < tokp");
730+
}
718731

719732
return lex_p > tokp;
720733
}

Diff for: ‎core/src/main/java/org/jruby/ext/ripper/StringTerm.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ private int endFound(RipperLexer lexer) throws IOException {
7676
if ((flags & STR_FUNC_QWORDS) != 0) {
7777
flags |= STR_FUNC_TERM;
7878
lexer.pushback(0);
79+
lexer.addDelayedToken(lexer.tokp, lexer.lex_p);
7980
return ' ';
8081
}
8182

@@ -118,7 +119,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
118119

119120
if ((flags & STR_FUNC_TERM) != 0) {
120121
if ((flags & STR_FUNC_QWORDS) != 0) lexer.nextc(); // delayed terminator char
121-
lexer.ignoreNextScanEvent = true;
122+
lexer.setState(EXPR_BEG | EXPR_LABEL);
122123
lexer.setStrTerm(null);
123124
return ((flags & STR_FUNC_REGEXP) != 0) ? RipperParser.tREGEXP_END : RipperParser.tSTRING_END;
124125
}
@@ -128,7 +129,6 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
128129
c = lexer.nextc();
129130
if ((flags & STR_FUNC_QWORDS) != 0 && Character.isWhitespace(c)) {
130131
do {
131-
buffer.append((char) c);
132132
c = lexer.nextc();
133133
} while (Character.isWhitespace(c));
134134
spaceSeen = true;
@@ -145,6 +145,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
145145

146146
if (spaceSeen) {
147147
lexer.pushback(c);
148+
lexer.addDelayedToken(lexer.tokp, lexer.lex_p);
148149
return ' ';
149150
}
150151

0 commit comments

Comments
 (0)
Please sign in to comment.