Skip to content

Commit

Permalink
zig fmt supports simple line comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewrk
andrewrk committed Feb 15, 2018
1 parent cc26148 commit 5f58809
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 24 deletions.
22 changes: 22 additions & 0 deletions std/zig/ast.zig
View file
Expand Up @@ -6,6 +6,7 @@ const mem = std.mem;

pub const Node = struct {
id: Id,
comment: ?&NodeLineComment,

pub const Id = enum {
Root,
Expand All @@ -20,6 +21,7 @@ pub const Node = struct {
FloatLiteral,
StringLiteral,
BuiltinCall,
LineComment,
};

pub fn iterate(base: &Node, index: usize) ?&Node {
Expand All @@ -36,6 +38,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
};
}

Expand All @@ -53,6 +56,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
};
}

Expand All @@ -70,6 +74,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
};
}
};
Expand Down Expand Up @@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct {
return self.token;
}
};

pub const NodeLineComment = struct {
base: Node,
lines: ArrayList(Token),

pub fn iterate(self: &NodeLineComment, index: usize) ?&Node {
return null;
}

pub fn firstToken(self: &NodeLineComment) Token {
return self.lines.at(0);
}

pub fn lastToken(self: &NodeLineComment) Token {
return self.lines.at(self.lines.len - 1);
}
};
83 changes: 70 additions & 13 deletions std/zig/parser.zig
View file
Expand Up @@ -18,6 +18,7 @@ pub const Parser = struct {
put_back_tokens: [2]Token,
put_back_count: usize,
source_file_name: []const u8,
pending_line_comment_node: ?&ast.NodeLineComment,

pub const Tree = struct {
root_node: &ast.NodeRoot,
Expand All @@ -43,6 +44,7 @@ pub const Parser = struct {
.put_back_count = 0,
.source_file_name = source_file_name,
.utility_bytes = []align(utility_bytes_align) u8{},
.pending_line_comment_node = null,
};
}

Expand Down Expand Up @@ -131,6 +133,33 @@ pub const Parser = struct {
// warn("\n");
//}

// look for line comments
while (true) {
const token = self.getNextToken();
if (token.id == Token.Id.LineComment) {
const node = blk: {
if (self.pending_line_comment_node) |comment_node| {
break :blk comment_node;
} else {
const comment_node = try arena.create(ast.NodeLineComment);
*comment_node = ast.NodeLineComment {
.base = ast.Node {
.id = ast.Node.Id.LineComment,
.comment = null,
},
.lines = ArrayList(Token).init(arena),
};
self.pending_line_comment_node = comment_node;
break :blk comment_node;
}
};
try node.lines.append(token);
continue;
}
self.putBackToken(token);
break;
}

// This gives us 1 free append that can't fail
const state = stack.pop();

Expand Down Expand Up @@ -329,7 +358,7 @@ pub const Parser = struct {
Token.Id.Builtin => {
const node = try arena.create(ast.NodeBuiltinCall);
*node = ast.NodeBuiltinCall {
.base = ast.Node {.id = ast.Node.Id.BuiltinCall},
.base = self.initNode(ast.Node.Id.BuiltinCall),
.builtin_token = token,
.params = ArrayList(&ast.Node).init(arena),
.rparen_token = undefined,
Expand All @@ -350,7 +379,7 @@ pub const Parser = struct {
Token.Id.StringLiteral => {
const node = try arena.create(ast.NodeStringLiteral);
*node = ast.NodeStringLiteral {
.base = ast.Node {.id = ast.Node.Id.StringLiteral},
.base = self.initNode(ast.Node.Id.StringLiteral),
.token = token,
};
try stack.append(State {
Expand All @@ -359,6 +388,7 @@ pub const Parser = struct {
try stack.append(State.AfterOperand);
continue;
},

else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
}
},
Expand Down Expand Up @@ -660,11 +690,19 @@ pub const Parser = struct {
}
}

fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
if (self.pending_line_comment_node) |comment_node| {
self.pending_line_comment_node = null;
return ast.Node {.id = id, .comment = comment_node};
}
return ast.Node {.id = id, .comment = null };
}

fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot {
const node = try arena.create(ast.NodeRoot);

*node = ast.NodeRoot {
.base = ast.Node {.id = ast.Node.Id.Root},
.base = self.initNode(ast.Node.Id.Root),
.decls = ArrayList(&ast.Node).init(arena),
// initialized when we get the eof token
.eof_token = undefined,
Expand All @@ -678,7 +716,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeVarDecl);

*node = ast.NodeVarDecl {
.base = ast.Node {.id = ast.Node.Id.VarDecl},
.base = self.initNode(ast.Node.Id.VarDecl),
.visib_token = *visib_token,
.mut_token = *mut_token,
.comptime_token = *comptime_token,
Expand All @@ -701,7 +739,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeFnProto);

*node = ast.NodeFnProto {
.base = ast.Node {.id = ast.Node.Id.FnProto},
.base = self.initNode(ast.Node.Id.FnProto),
.visib_token = *visib_token,
.name_token = null,
.fn_token = *fn_token,
Expand All @@ -722,7 +760,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeParamDecl);

*node = ast.NodeParamDecl {
.base = ast.Node {.id = ast.Node.Id.ParamDecl},
.base = self.initNode(ast.Node.Id.ParamDecl),
.comptime_token = null,
.noalias_token = null,
.name_token = null,
Expand All @@ -736,7 +774,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeBlock);

*node = ast.NodeBlock {
.base = ast.Node {.id = ast.Node.Id.Block},
.base = self.initNode(ast.Node.Id.Block),
.begin_token = *begin_token,
.end_token = undefined,
.statements = ArrayList(&ast.Node).init(arena),
Expand All @@ -748,7 +786,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeInfixOp);

*node = ast.NodeInfixOp {
.base = ast.Node {.id = ast.Node.Id.InfixOp},
.base = self.initNode(ast.Node.Id.InfixOp),
.op_token = *op_token,
.lhs = undefined,
.op = *op,
Expand All @@ -761,7 +799,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodePrefixOp);

*node = ast.NodePrefixOp {
.base = ast.Node {.id = ast.Node.Id.PrefixOp},
.base = self.initNode(ast.Node.Id.PrefixOp),
.op_token = *op_token,
.op = *op,
.rhs = undefined,
Expand All @@ -773,7 +811,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeIdentifier);

*node = ast.NodeIdentifier {
.base = ast.Node {.id = ast.Node.Id.Identifier},
.base = self.initNode(ast.Node.Id.Identifier),
.name_token = *name_token,
};
return node;
Expand All @@ -783,7 +821,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeIntegerLiteral);

*node = ast.NodeIntegerLiteral {
.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
.base = self.initNode(ast.Node.Id.IntegerLiteral),
.token = *token,
};
return node;
Expand All @@ -793,7 +831,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeFloatLiteral);

*node = ast.NodeFloatLiteral {
.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
.base = self.initNode(ast.Node.Id.FloatLiteral),
.token = *token,
};
return node;
Expand Down Expand Up @@ -1158,9 +1196,11 @@ pub const Parser = struct {
}
}
},
ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),

ast.Node.Id.Root,
ast.Node.Id.VarDecl,
ast.Node.Id.FnProto,
ast.Node.Id.ParamDecl => unreachable,
},
RenderState.FnProtoRParen => |fn_proto| {
Expand All @@ -1187,6 +1227,12 @@ pub const Parser = struct {
}
},
RenderState.Statement => |base| {
if (base.comment) |comment| {
for (comment.lines.toSliceConst()) |line_token| {
try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
try stream.writeByteNTimes(' ', indent);
}
}
switch (base.id) {
ast.Node.Id.VarDecl => {
const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
Expand Down Expand Up @@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void {
}

test "zig fmt" {
try testCanonical(
\\const std = @import("std");
\\
\\pub fn main() !void {
\\ // If this program is run without stdout attached, exit with an error.
\\ // another comment
\\ var stdout_file = try std.io.getStdOut;
\\}
\\
);

try testCanonical(
\\const std = @import("std");
\\
Expand Down
14 changes: 3 additions & 11 deletions std/zig/tokenizer.zig
View file
Expand Up @@ -99,6 +99,7 @@ pub const Token = struct {
AmpersandEqual,
IntegerLiteral,
FloatLiteral,
LineComment,
Keyword_align,
Keyword_and,
Keyword_asm,
Expand Down Expand Up @@ -470,7 +471,7 @@ pub const Tokenizer = struct {

State.Slash => switch (c) {
'/' => {
result.id = undefined;
result.id = Token.Id.LineComment;
state = State.LineComment;
},
else => {
Expand All @@ -479,16 +480,7 @@ pub const Tokenizer = struct {
},
},
State.LineComment => switch (c) {
'\n' => {
state = State.Start;
result = Token {
.id = Token.Id.Eof,
.start = self.index + 1,
.column = 0,
.line = self.line + 1,
.end = undefined,
};
},
'\n' => break,
else => self.checkLiteralCharacter(),
},
State.Zero => switch (c) {
Expand Down

0 comments on commit 5f58809

Please sign in to comment.