Add very initial unicode escape sequence parsing in strings (#437)

Currently only ruby >= 1.9 can handle unicode, due to a reliance on Encoding support missing from ruby 1.8.x
opal · Dec 3, 2013 · dc98d0c · dc98d0c
1 parent 6ca583c
commit dc98d0c
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,7 @@
 *   Fix parsing of escapes in single-strings ('foo\n'). Only ' and \
     characters now get escaped in single quoted strings. Also, more escape
     sequences in double-quoted strings are now supported: `\a`, `\v`, `\f`,
-    `\e`, `\s`, octal (`\314`), hex (`\xff`).
+    `\e`, `\s`, octal (`\314`), hex (`\xff`) and unicode (`\u1234`).
 
 *   Sourcemaps revamp. Lexer now tracks column and line info for ever token to
     produce much more accurate sourcemaps. All method calls are now located on

diff --git a/lib/opal/parser/lexer.rb b/lib/opal/parser/lexer.rb
@@ -210,6 +210,13 @@ def read_escape
         (matched.to_i(8) % 0x100).chr
       elsif scan(/x([0-9a-fA-F]{1,2})/)
         scanner[1].to_i(16).chr
+      elsif scan(/u([0-9a-zA-Z]{1,4})/)
+        if defined?(::Encoding)
+          scanner[1].to_i(16).chr(Encoding::UTF_8)
+        else
+          # FIXME: no encoding on 1.8
+          ""
+        end
       else
         # escaped char doesnt need escaping, so just return it
         scan(/./)

diff --git a/spec/cli/lexer_spec.rb b/spec/cli/lexer_spec.rb
@@ -69,5 +69,11 @@
     it "can parse hex escape sequences" do
       expect_parsed_string('"\\x61\\x63\\x62"').to eq('acb')
     end
+
+    if defined?(::Encoding)
+      it "can parse simple unicode sequences" do
+        expect_parsed_string('"\\u1234"').to eq("\u1234")
+      end
+    end
   end
 end