Use our thread-local UTF8 decoder to avoid constructing any.

jruby · May 21, 2015 · 6c53669 · 6c53669
1 parent f8782ac
commit 6c53669
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java b/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
@@ -46,6 +46,7 @@
 import org.joni.Option;
 import org.joni.Regex;
 import org.jruby.Ruby;
+import org.jruby.RubyEncoding;
 import org.jruby.RubyRegexp;
 import org.jruby.ast.BackRefNode;
 import org.jruby.ast.BignumNode;
@@ -314,7 +315,13 @@ public String createTokenString() {
         // FIXME: We should be able to move some faster non-exception cache using Encoding.isDefined
         try {
             charset = current_enc.getCharset();
-            if (charset != null) return new String(bytes, begin + tokp, lex_p - tokp, charset);
+            if (charset != null) {
+                if (charset == RubyEncoding.UTF8) {
+                    return RubyEncoding.decodeUTF8(bytes, begin + tokp, lex_p - tokp);
+                } else {
+                    return new String(bytes, begin + tokp, lex_p - tokp, charset);
+                }
+            }
         } catch (UnsupportedCharsetException e) {}