zig fmt: fix tokenization of float literal with exponent

ziglang · May 4, 2018 · 8721eb6 · 8721eb6
1 parent ef3111b
commit 8721eb6
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 59 deletions.
diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig
@@ -17,13 +17,12 @@
 //}
 
 
-//TODO
-//test "zig fmt: number literals" {
-//    try testCanonical(
-//        \\pub const f64_true_min = 4.94065645841246544177e-324;
-//        \\
-//    );
-//}
+test "zig fmt: float literal with exponent" {
+    try testCanonical(
+        \\pub const f64_true_min = 4.94065645841246544177e-324;
+        \\
+    );
+}
 
 test "zig fmt: line comments in struct initializer" {
     try testCanonical(

diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig
@@ -6,60 +6,60 @@ pub const Token = struct {
     start: usize,
     end: usize,
 
-    const KeywordId = struct {
+    const Keyword = struct {
         bytes: []const u8,
         id: Id,
     };
 
-    const keywords = []KeywordId {
-        KeywordId{.bytes="align", .id = Id.Keyword_align},
-        KeywordId{.bytes="and", .id = Id.Keyword_and},
-        KeywordId{.bytes="asm", .id = Id.Keyword_asm},
-        KeywordId{.bytes="async", .id = Id.Keyword_async},
-        KeywordId{.bytes="await", .id = Id.Keyword_await},
-        KeywordId{.bytes="break", .id = Id.Keyword_break},
-        KeywordId{.bytes="catch", .id = Id.Keyword_catch},
-        KeywordId{.bytes="cancel", .id = Id.Keyword_cancel},
-        KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
-        KeywordId{.bytes="const", .id = Id.Keyword_const},
-        KeywordId{.bytes="continue", .id = Id.Keyword_continue},
-        KeywordId{.bytes="defer", .id = Id.Keyword_defer},
-        KeywordId{.bytes="else", .id = Id.Keyword_else},
-        KeywordId{.bytes="enum", .id = Id.Keyword_enum},
-        KeywordId{.bytes="errdefer", .id = Id.Keyword_errdefer},
-        KeywordId{.bytes="error", .id = Id.Keyword_error},
-        KeywordId{.bytes="export", .id = Id.Keyword_export},
-        KeywordId{.bytes="extern", .id = Id.Keyword_extern},
-        KeywordId{.bytes="false", .id = Id.Keyword_false},
-        KeywordId{.bytes="fn", .id = Id.Keyword_fn},
-        KeywordId{.bytes="for", .id = Id.Keyword_for},
-        KeywordId{.bytes="if", .id = Id.Keyword_if},
-        KeywordId{.bytes="inline", .id = Id.Keyword_inline},
-        KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
-        KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
-        KeywordId{.bytes="null", .id = Id.Keyword_null},
-        KeywordId{.bytes="or", .id = Id.Keyword_or},
-        KeywordId{.bytes="packed", .id = Id.Keyword_packed},
-        KeywordId{.bytes="promise", .id = Id.Keyword_promise},
-        KeywordId{.bytes="pub", .id = Id.Keyword_pub},
-        KeywordId{.bytes="resume", .id = Id.Keyword_resume},
-        KeywordId{.bytes="return", .id = Id.Keyword_return},
-        KeywordId{.bytes="section", .id = Id.Keyword_section},
-        KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
-        KeywordId{.bytes="struct", .id = Id.Keyword_struct},
-        KeywordId{.bytes="suspend", .id = Id.Keyword_suspend},
-        KeywordId{.bytes="switch", .id = Id.Keyword_switch},
-        KeywordId{.bytes="test", .id = Id.Keyword_test},
-        KeywordId{.bytes="this", .id = Id.Keyword_this},
-        KeywordId{.bytes="true", .id = Id.Keyword_true},
-        KeywordId{.bytes="try", .id = Id.Keyword_try},
-        KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
-        KeywordId{.bytes="union", .id = Id.Keyword_union},
-        KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
-        KeywordId{.bytes="use", .id = Id.Keyword_use},
-        KeywordId{.bytes="var", .id = Id.Keyword_var},
-        KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
-        KeywordId{.bytes="while", .id = Id.Keyword_while},
+    const keywords = []Keyword {
+        Keyword{.bytes="align", .id = Id.Keyword_align},
+        Keyword{.bytes="and", .id = Id.Keyword_and},
+        Keyword{.bytes="asm", .id = Id.Keyword_asm},
+        Keyword{.bytes="async", .id = Id.Keyword_async},
+        Keyword{.bytes="await", .id = Id.Keyword_await},
+        Keyword{.bytes="break", .id = Id.Keyword_break},
+        Keyword{.bytes="catch", .id = Id.Keyword_catch},
+        Keyword{.bytes="cancel", .id = Id.Keyword_cancel},
+        Keyword{.bytes="comptime", .id = Id.Keyword_comptime},
+        Keyword{.bytes="const", .id = Id.Keyword_const},
+        Keyword{.bytes="continue", .id = Id.Keyword_continue},
+        Keyword{.bytes="defer", .id = Id.Keyword_defer},
+        Keyword{.bytes="else", .id = Id.Keyword_else},
+        Keyword{.bytes="enum", .id = Id.Keyword_enum},
+        Keyword{.bytes="errdefer", .id = Id.Keyword_errdefer},
+        Keyword{.bytes="error", .id = Id.Keyword_error},
+        Keyword{.bytes="export", .id = Id.Keyword_export},
+        Keyword{.bytes="extern", .id = Id.Keyword_extern},
+        Keyword{.bytes="false", .id = Id.Keyword_false},
+        Keyword{.bytes="fn", .id = Id.Keyword_fn},
+        Keyword{.bytes="for", .id = Id.Keyword_for},
+        Keyword{.bytes="if", .id = Id.Keyword_if},
+        Keyword{.bytes="inline", .id = Id.Keyword_inline},
+        Keyword{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
+        Keyword{.bytes="noalias", .id = Id.Keyword_noalias},
+        Keyword{.bytes="null", .id = Id.Keyword_null},
+        Keyword{.bytes="or", .id = Id.Keyword_or},
+        Keyword{.bytes="packed", .id = Id.Keyword_packed},
+        Keyword{.bytes="promise", .id = Id.Keyword_promise},
+        Keyword{.bytes="pub", .id = Id.Keyword_pub},
+        Keyword{.bytes="resume", .id = Id.Keyword_resume},
+        Keyword{.bytes="return", .id = Id.Keyword_return},
+        Keyword{.bytes="section", .id = Id.Keyword_section},
+        Keyword{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
+        Keyword{.bytes="struct", .id = Id.Keyword_struct},
+        Keyword{.bytes="suspend", .id = Id.Keyword_suspend},
+        Keyword{.bytes="switch", .id = Id.Keyword_switch},
+        Keyword{.bytes="test", .id = Id.Keyword_test},
+        Keyword{.bytes="this", .id = Id.Keyword_this},
+        Keyword{.bytes="true", .id = Id.Keyword_true},
+        Keyword{.bytes="try", .id = Id.Keyword_try},
+        Keyword{.bytes="undefined", .id = Id.Keyword_undefined},
+        Keyword{.bytes="union", .id = Id.Keyword_union},
+        Keyword{.bytes="unreachable", .id = Id.Keyword_unreachable},
+        Keyword{.bytes="use", .id = Id.Keyword_use},
+        Keyword{.bytes="var", .id = Id.Keyword_var},
+        Keyword{.bytes="volatile", .id = Id.Keyword_volatile},
+        Keyword{.bytes="while", .id = Id.Keyword_while},
     };
 
     fn getKeyword(bytes: []const u8) ?Id {
@@ -912,10 +912,10 @@ pub const Tokenizer = struct {
                     },
                 },
                 State.FloatFraction => switch (c) {
-                    'p', 'P' => {
+                    'p', 'P', 'e', 'E' => {
                         state = State.FloatExponentUnsigned;
                     },
-                    '0'...'9', 'a'...'f', 'A'...'F' => {},
+                    '0'...'9' => {},
                     else => break,
                 },
                 State.FloatExponentUnsigned => switch (c) {
@@ -1108,6 +1108,15 @@ test "tokenizer" {
     });
 }
 
+test "tokenizer - float literal" {
+    testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
+        Token.Id.Identifier,
+        Token.Id.Equal,
+        Token.Id.FloatLiteral,
+        Token.Id.Semicolon,
+    });
+}
+
 test "tokenizer - chars" {
     testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
 }