Skip to content

Commit

Permalink
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@
import org.joni.Option;
import org.joni.Regex;
import org.jruby.Ruby;
import org.jruby.RubyEncoding;
import org.jruby.RubyRegexp;
import org.jruby.ast.BackRefNode;
import org.jruby.ast.BignumNode;
@@ -314,7 +315,13 @@ public String createTokenString() {
// FIXME: We should be able to move some faster non-exception cache using Encoding.isDefined
try {
charset = current_enc.getCharset();
if (charset != null) return new String(bytes, begin + tokp, lex_p - tokp, charset);
if (charset != null) {
if (charset == RubyEncoding.UTF8) {
return RubyEncoding.decodeUTF8(bytes, begin + tokp, lex_p - tokp);
} else {
return new String(bytes, begin + tokp, lex_p - tokp, charset);
}
}
} catch (UnsupportedCharsetException e) {}


0 comments on commit 6c53669

Please sign in to comment.