diff --git a/crates/compiler/builtins/bitcode/build.zig b/crates/compiler/builtins/bitcode/build.zig index fb76a5de619..7c51dbb6804 100644 --- a/crates/compiler/builtins/bitcode/build.zig +++ b/crates/compiler/builtins/bitcode/build.zig @@ -119,9 +119,9 @@ fn generateObjectFile( const suffix = if (target.result.os.tag == std.Target.Os.Tag.windows) - "obj" - else - "o"; + "obj" + else + "o"; const install = b.addInstallFile(obj_file, b.fmt("{s}.{s}", .{ object_name, suffix })); const obj_step = b.step(step_name, "Build object file for linking"); diff --git a/src/base/sexpr.zig b/src/base/sexpr.zig index e07a7b8902b..46ae9d72250 100644 --- a/src/base/sexpr.zig +++ b/src/base/sexpr.zig @@ -129,7 +129,7 @@ pub const Expr = union(enum) { try writer.print(")", .{}); }, - .string => |s| try writer.print("'{s}'", .{s}), + .string => |s| try writer.print("\"{s}\"", .{s}), .signed_int => |i| try writer.print("{d}", .{i}), .unsigned_int => |u| try writer.print("{d}", .{u}), .float => |f| try writer.print("{any}", .{f}), @@ -191,7 +191,7 @@ test "s-expression" { foo.toStringPretty(buf.writer().any()); const expected = \\(foo - \\ 'bar' + \\ "bar" \\ -123 \\ (baz 456 7.89e2)) ; diff --git a/src/check/parse.zig b/src/check/parse.zig index 32e18f1fe20..a8b1a0ff591 100644 --- a/src/check/parse.zig +++ b/src/check/parse.zig @@ -22,8 +22,8 @@ pub fn parse(env: *base.ModuleEnv, source: []const u8) IR { tokenizer.tokenize(); const result = tokenizer.finishAndDeinit(); - if (result.messages.len > 0) { - tokenizeReport(env.gpa, source, result.messages); + for (result.messages) |msg| { + _ = env.problems.append(env.gpa, .{ .tokenize = msg }); } var parser = Parser.init(result.tokens); @@ -31,6 +31,10 @@ pub fn parse(env: *base.ModuleEnv, source: []const u8) IR { parser.parseFile(); + for (parser.diagnostics.items) |msg| { + _ = env.problems.append(env.gpa, .{ .parser = msg }); + } + const errors = parser.diagnostics.toOwnedSlice(env.gpa) catch |err| exitOnOom(err); return .{ @@ -40,106 +44,3 @@ pub fn parse(env: *base.ModuleEnv, source: []const u8) IR { .errors = errors, }; } - -fn lineNum(newlines: std.ArrayList(usize), pos: u32) u32 { - const pos_usize = @as(usize, @intCast(pos)); - var lineno: u32 = 0; - while (lineno < newlines.items.len) { - if (newlines.items[lineno + 1] > pos_usize) { - return lineno; - } - lineno += 1; - } - return lineno; -} - -fn tokenizeReport(allocator: std.mem.Allocator, source: []const u8, msgs: []const tokenize.Diagnostic) void { - std.debug.print("Found the {d} following issues while tokenizing:\n", .{msgs.len}); - var newlines = std.ArrayList(usize).init(allocator); - defer newlines.deinit(); - newlines.append(0) catch |err| exitOnOom(err); - var pos: usize = 0; - for (source) |c| { - if (c == '\n') { - newlines.append(pos) catch |err| exitOnOom(err); - } - pos += 1; - } - for (msgs) |message| { - switch (message.tag) { - .MismatchedBrace => { - const start_line_num = lineNum(newlines, message.begin); - const start_col = message.begin - newlines.items[start_line_num]; - const end_line_num = lineNum(newlines, message.end); - const end_col = message.end - newlines.items[end_line_num]; - - const src = source[newlines.items[start_line_num]..newlines.items[end_line_num + 1]]; - var spaces = std.ArrayList(u8).init(allocator); - defer spaces.deinit(); - for (0..start_col) |_| { - spaces.append(' ') catch |err| exitOnOom(err); - } - - std.debug.print( - "({d}:{d}-{d}:{d}) Expected the correct closing brace here:\n{s}\n{s}^\n", - .{ start_line_num, start_col, end_line_num, end_col, src, spaces.toOwnedSlice() catch |err| exitOnOom(err) }, - ); - }, - else => { - std.debug.print("MSG: {any}\n", .{message}); - }, - } - } -} - -// TODO move this somewhere better, for now it's here to keep it simple. -fn testSExprHelper(source: []const u8, expected: []const u8) !void { - var env = base.ModuleEnv.init(testing.allocator); - defer env.deinit(); - - // parse our source - var parse_ast = parse(&env, source); - defer parse_ast.deinit(); - std.testing.expectEqualSlices(IR.Diagnostic, &[_]IR.Diagnostic{}, parse_ast.errors) catch { - std.debug.print("Tokens:\n{any}", .{parse_ast.tokens.tokens.items(.tag)}); - std.debug.panic("Test failed with parse errors", .{}); - }; - - // shouldn't be required in future - parse_ast.store.emptyScratch(); - - // buffer to write our SExpr to - var buf = std.ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - - // convert the AST to our SExpr - try parse_ast.toSExprStr(&env, buf.writer().any()); - - // TODO in future we should just write the SExpr to a file and snapshot it - // for now we are comparing strings to keep it simple - try testing.expectEqualStrings(expected, buf.items[0..]); -} - -test "example s-expr" { - const source = - \\module [foo, bar] - \\ - \\foo = "hey" - \\bar = "yo" - ; - - const expected = - \\(file - \\ (header - \\ (exposed_item (lower_ident 'foo')) - \\ (exposed_item (lower_ident 'bar'))) - \\ (decl - \\ (ident 'foo') - \\ (string 'hey')) - \\ (decl - \\ (ident 'bar') - \\ (string 'yo'))) - ; - - try testSExprHelper(source, expected); -} diff --git a/src/check/parse/IR.zig b/src/check/parse/IR.zig index f7b4b6a71f0..15ab7d8c6d4 100644 --- a/src/check/parse/IR.zig +++ b/src/check/parse/IR.zig @@ -53,6 +53,7 @@ pub fn regionIsMultiline(self: *IR, region: Region) bool { pub fn deinit(self: *IR) void { defer self.tokens.deinit(); defer self.store.deinit(); + self.store.gpa.free(self.errors); } /// Diagnostics related to parsing @@ -78,6 +79,17 @@ pub const Diagnostic = struct { expected_platform_string, expected_package_or_platform_string, expected_package_platform_close_curly, + expect_closing_paren, + header_expected_open_square, + header_expected_close_square, + header_unexpected_token, + pattern_unexpected_token, + ty_anno_unexpected_token, + statement_unexpected_eof, + statement_unexpected_token, + string_unexpected_token, + expr_if_missing_else, + expr_no_space_dot_int, import_exposing_no_open, import_exposing_no_close, no_else, @@ -648,6 +660,9 @@ pub const NodeStore = struct { node.data.rhs = mod.exposes.span.len; node.region = mod.region; }, + .malformed => { + @panic("use addMalformed instead"); + }, else => {}, } const nid = store.nodes.append(store.gpa, node); @@ -776,6 +791,9 @@ pub const NodeStore = struct { node.data.lhs = a.name; node.data.rhs = a.anno.id; }, + .malformed => { + @panic("use addMalformed instead"); + }, } const nid = store.nodes.append(store.gpa, node); return .{ .id = @intFromEnum(nid) }; @@ -845,12 +863,17 @@ pub const NodeStore = struct { node.region = u.region; }, .alternatives => |a| { - std.debug.assert(a.patterns.span.len > 1); + // disabled because it was hit by a fuzz test + // for a repro see src/snapshots/fuzz_crash_012.txt + // std.debug.assert(a.patterns.span.len > 1); node.region = a.region; node.tag = .alternatives_patt; node.data.lhs = a.patterns.span.start; node.data.rhs = a.patterns.span.len; }, + .malformed => { + @panic("use addMalformed instead"); + }, } const nid = store.nodes.append(store.gpa, node); return .{ .id = @intFromEnum(nid) }; @@ -997,6 +1020,9 @@ pub const NodeStore = struct { node.tag = .ellipsis; node.region = e.region; }, + .malformed => { + @panic("use addMalformed instead"); + }, } const nid = store.nodes.append(store.gpa, node); return .{ .id = @intFromEnum(nid) }; @@ -1168,6 +1194,9 @@ pub const NodeStore = struct { node.region = p.region; node.data.lhs = p.anno.id; }, + .malformed => { + @panic("use addMalformed instead"); + }, } const nid = store.nodes.append(store.gpa, node); @@ -1224,6 +1253,9 @@ pub const NodeStore = struct { .region = node.region, } }; }, + .malformed => { + return .{ .malformed = .{ .reason = @enumFromInt(node.data.lhs) } }; + }, else => { std.debug.panic("Expected a valid header tag, got {s}", .{@tagName(node.tag)}); }, @@ -1597,6 +1629,9 @@ pub const NodeStore = struct { .region = node.region, } }; }, + .malformed => { + return .{ .malformed = .{ .reason = @enumFromInt(node.data.lhs) } }; + }, else => { std.debug.panic("Expected a valid expr tag, got {s}", .{@tagName(node.tag)}); }, @@ -1719,6 +1754,9 @@ pub const NodeStore = struct { .anno = .{ .id = node.data.lhs }, } }; }, + .malformed => { + return .{ .malformed = .{ .reason = @enumFromInt(node.data.lhs) } }; + }, else => { std.debug.panic("Expected a valid type annotation node, found {s}", .{@tagName(node.tag)}); }, @@ -1803,13 +1841,21 @@ pub const NodeStore = struct { // TODO: complete this region: Region, }, + malformed: struct { + reason: Diagnostic.Tag, + }, const AppHeaderRhs = packed struct { num_packages: u10, num_provides: u22 }; pub fn toSExpr(self: @This(), env: *base.ModuleEnv, ir: *IR) sexpr.Expr { switch (self) { + .app => { + var node = sexpr.Expr.init(env.gpa, "app"); + node.appendStringChild(env.gpa, "TODO implement toSExpr for app module header"); + return node; + }, .module => |module| { - var header_node = sexpr.Expr.init(env.gpa, "header"); + var header_node = sexpr.Expr.init(env.gpa, "module"); for (ir.store.exposedItemSlice(module.exposes)) |exposed| { const item = ir.store.getExposedItem(exposed); @@ -1819,7 +1865,26 @@ pub const NodeStore = struct { return header_node; }, - else => @panic("not implemented"), + .package => { + var node = sexpr.Expr.init(env.gpa, "package"); + node.appendStringChild(env.gpa, "TODO implement toSExpr for package module header"); + return node; + }, + .platform => { + var node = sexpr.Expr.init(env.gpa, "platform"); + node.appendStringChild(env.gpa, "TODO implement toSExpr for platform module header"); + return node; + }, + .hosted => { + var node = sexpr.Expr.init(env.gpa, "hosted"); + node.appendStringChild(env.gpa, "TODO implement toSExpr for hosted module header"); + return node; + }, + .malformed => |a| { + var node = sexpr.Expr.init(env.gpa, "malformed_header"); + node.appendStringChild(env.gpa, @tagName(a.reason)); + return node; + }, } } }; @@ -1909,6 +1974,9 @@ pub const NodeStore = struct { anno: TypeAnnoIdx, region: Region, }, + malformed: struct { + reason: Diagnostic.Tag, + }, pub const Import = struct { module_name_tok: TokenIdx, @@ -2026,6 +2094,9 @@ pub const NodeStore = struct { node.appendNodeChild(env.gpa, &child); return node; }, + else => { + std.debug.panic("implement toSExpr for Statement: {}", .{self}); + }, } } }; @@ -2071,6 +2142,9 @@ pub const NodeStore = struct { anno: TypeAnnoIdx, region: Region, }, + malformed: struct { + reason: Diagnostic.Tag, + }, const TagUnionRhs = packed struct { open: u1, tags_len: u31 }; @@ -2140,6 +2214,11 @@ pub const NodeStore = struct { .parens => |a| { return ir.store.getTypeAnno(a.anno).toSExpr(env, ir); }, + .malformed => |a| { + var node = sexpr.Expr.init(env.gpa, "malformed_expr"); + node.appendStringChild(env.gpa, @tagName(a.reason)); + return node; + }, } } }; @@ -2193,6 +2272,9 @@ pub const NodeStore = struct { patterns: PatternSpan, region: Region, }, + malformed: struct { + reason: Diagnostic.Tag, + }, pub fn toSExpr(self: @This(), env: *base.ModuleEnv, ir: *IR) sexpr.Expr { switch (self) { @@ -2203,7 +2285,46 @@ pub const NodeStore = struct { return node; }, - else => @panic("formatting for this pattern not yet implemented"), + .tag => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .number => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .string => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .record => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .list => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .list_rest => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .tuple => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .underscore => { + return sexpr.Expr.init(env.gpa, "underscore"); + }, + .alternatives => { + std.debug.print("TODO implement toSExpr for Pattern {} not yet implemented", .{self}); + @panic("unimplemented"); + }, + .malformed => |a| { + var node = sexpr.Expr.init(env.gpa, "malformed_pattern"); + node.appendStringChild(env.gpa, @tagName(a.reason)); + return node; + }, } } }; @@ -2290,6 +2411,9 @@ pub const NodeStore = struct { region: Region, }, block: Body, + malformed: struct { + reason: Diagnostic.Tag, + }, pub fn as_string_part_region(self: @This()) !Region { switch (self) { @@ -2328,19 +2452,6 @@ pub const NodeStore = struct { .block => |block| { return block.toSExpr(env, ir); }, - // (binop ) - .bin_op => |binop| { - var binop_sexpr = sexpr.Expr.init(env.gpa, "binop"); - - binop_sexpr.appendStringChild(env.gpa, ir.resolve(binop.operator)); - - var lhs = ir.store.getExpr(binop.left).toSExpr(env, ir); - var rhs = ir.store.getExpr(binop.right).toSExpr(env, ir); - binop_sexpr.appendNodeChild(env.gpa, &lhs); - binop_sexpr.appendNodeChild(env.gpa, &rhs); - - return binop_sexpr; - }, // (if_then_else ) .if_then_else => |stmt| { var node = sexpr.Expr.init(env.gpa, "if_then_else"); @@ -2361,8 +2472,100 @@ pub const NodeStore = struct { ident_sexpr.appendStringChild(env.gpa, ir.resolve(ident.token)); return ident_sexpr; }, + // (list []) + .list => |a| { + var node = sexpr.Expr.init(env.gpa, "list"); + for (ir.store.exprSlice(a.items)) |b| { + var child = ir.store.getExpr(b).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &child); + } + return node; + }, + // (malformed_expr ) + .malformed => |a| { + var node = sexpr.Expr.init(env.gpa, "malformed_expr"); + node.appendStringChild(env.gpa, @tagName(a.reason)); + return node; + }, + // (float ) + .float => |a| { + var node = sexpr.Expr.init(env.gpa, "float"); + node.appendStringChild(env.gpa, ir.resolve(a.token)); + return node; + }, + // (tuple []) + .tuple => |a| { + var node = sexpr.Expr.init(env.gpa, "tuple"); + + for (ir.store.exprSlice(a.items)) |item| { + var child = ir.store.getExpr(item).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &child); + } + + return node; + }, + // (record [(field ?optional)]) + .record => |a| { + var node = sexpr.Expr.init(env.gpa, "record"); + + for (ir.store.recordFieldSlice(a.fields)) |field_idx| { + const record_field = ir.store.getRecordField(field_idx); + var record_field_node = sexpr.Expr.init(env.gpa, "field"); + record_field_node.appendStringChild(env.gpa, ir.resolve(record_field.name)); + if (record_field.value != null) { + var value_node = ir.store.getExpr(record_field.value.?).toSExpr(env, ir); + record_field_node.appendNodeChild(env.gpa, &value_node); + } + if (record_field.optional) { + record_field_node.appendStringChild(env.gpa, "optional"); + } + node.appendNodeChild(env.gpa, &record_field_node); + } + + return node; + }, + // (apply []) + .apply => |a| { + var node = sexpr.Expr.init(env.gpa, "apply"); + var apply_fn = ir.store.getExpr(a.@"fn").toSExpr(env, ir); + node.appendNodeChild(env.gpa, &apply_fn); + + for (ir.store.exprSlice(a.args)) |arg| { + var arg_node = ir.store.getExpr(arg).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &arg_node); + } + + return node; + }, + .field_access => |a| { + var node = sexpr.Expr.init(env.gpa, "field_access"); + var child = a.toSExpr(env, ir); + node.appendNodeChild(env.gpa, &child); + return node; + }, + // (binop ) + .bin_op => |a| { + return a.toSExpr(env, ir); + }, + .lambda => |a| { + var node = sexpr.Expr.init(env.gpa, "lambda"); + + // arguments + var args = sexpr.Expr.init(env.gpa, "args"); + for (ir.store.patternSlice(a.args)) |arg| { + var arg_node = ir.store.getPattern(arg).toSExpr(env, ir); + args.appendNodeChild(env.gpa, &arg_node); + } + node.appendNodeChild(env.gpa, &args); + + // body + var body = ir.store.getExpr(a.body).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &body); + + return node; + }, else => { - std.debug.print("Format for Expr {}", .{self}); + std.debug.print("\n\n toSExpr not implement for Expr {}\n\n", .{self}); @panic("not implemented yet"); }, } @@ -2400,6 +2603,19 @@ pub const NodeStore = struct { right: ExprIdx, operator: TokenIdx, region: Region, + + /// (binop ) e.g. (binop '+' 1 2) + pub fn toSExpr(self: *const @This(), env: *base.ModuleEnv, ir: *IR) sexpr.Expr { + var node = sexpr.Expr.init(env.gpa, "binop"); + node.appendStringChild(env.gpa, ir.resolve(self.operator)); + + var left = ir.store.getExpr(self.left).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &left); + + var right = ir.store.getExpr(self.right).toSExpr(env, ir); + node.appendNodeChild(env.gpa, &right); + return node; + } }; pub const Unary = struct { diff --git a/src/check/parse/Parser.zig b/src/check/parse/Parser.zig index 0bddb07093f..73e61423719 100644 --- a/src/check/parse/Parser.zig +++ b/src/check/parse/Parser.zig @@ -116,7 +116,9 @@ pub fn pushDiagnostic(self: *Parser, tag: IR.Diagnostic.Tag, region: IR.Region) /// add a malformed token pub fn pushMalformed(self: *Parser, comptime t: type, tag: IR.Diagnostic.Tag, start: TokenIdx) t { const pos = self.pos; - self.advanceOne(); // TODO: find a better point to advance to + if (self.peek() != .EndOfFile) { + self.advanceOne(); // TODO: find a better point to advance to + } const region = IR.Region{ .start = start, .end = pos }; self.diagnostics.append(self.gpa, .{ .tag = tag, @@ -142,9 +144,6 @@ pub fn parseFile(self: *Parser) void { const scratch_top = self.store.scratchStatementTop(); while (self.peek() != .EndOfFile) { - if (self.peek() == .EndOfFile) { - break; - } const current_scratch_top = self.store.scratchStatementTop(); if (self.parseStmt()) |idx| { std.debug.assert(self.store.scratchStatementTop() == current_scratch_top); @@ -155,8 +154,6 @@ pub fn parseFile(self: *Parser) void { } } - std.debug.assert(self.store.scratch_statements.items.len > 0); - _ = self.store.addFile(.{ .header = header, .statements = self.store.statementSpanFrom(scratch_top), @@ -166,7 +163,7 @@ pub fn parseFile(self: *Parser) void { fn parseCollection(self: *Parser, comptime T: type, end_token: Token.Tag, scratch: *std.ArrayListUnmanaged(T), parser: fn (*Parser) T) ExpectError!usize { const scratch_top = scratch.items.len; - while (self.peek() != end_token) { + while (self.peek() != end_token and self.peek() != .EndOfFile) { scratch.append(self.gpa, parser(self)) catch |err| exitOnOom(err); self.expect(.Comma) catch { break; @@ -179,6 +176,7 @@ fn parseCollection(self: *Parser, comptime T: type, end_token: Token.Tag, scratc } /// Parses the items of type T until we encounter end_token, with each item separated by a Comma token +/// /// Returns the ending position of the collection fn parseCollectionSpan(self: *Parser, comptime T: type, end_token: Token.Tag, scratch_fn: fn (*IR.NodeStore, T) void, parser: fn (*Parser) T) ExpectError!u32 { while (self.peek() != end_token) { @@ -224,14 +222,16 @@ fn parseModuleHeader(self: *Parser) IR.NodeStore.HeaderIdx { // Get exposes self.expect(.OpenSquare) catch { - std.debug.panic("TODO: Handle header with no exposes open bracket: {s}", .{@tagName(self.peek())}); + return self.pushMalformed(IR.NodeStore.HeaderIdx, .header_expected_open_square, self.pos); }; const scratch_top = self.store.scratchExposedItemTop(); _ = self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { self.advance(); } - self.expect(.CloseSquare) catch {}; + self.expect(.CloseSquare) catch { + return self.pushMalformed(IR.NodeStore.HeaderIdx, .header_expected_close_square, start); + }; self.store.clearScratchExposedItemsFrom(scratch_top); return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close, start); }; @@ -263,7 +263,9 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { self.advance(); } - self.expect(.CloseSquare) catch {}; + self.expect(.CloseSquare) catch { + return self.pushMalformed(IR.NodeStore.HeaderIdx, .header_expected_close_square, start); + }; self.store.clearScratchExposedItemsFrom(scratch_top); return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close, start); }; @@ -274,7 +276,8 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { self.expect(.OpenCurly) catch { return self.pushMalformed(IR.NodeStore.HeaderIdx, .expected_package_platform_open_curly, start); }; - while (self.peek() != .CloseCurly) { + + while (self.peek() != .CloseCurly and self.peek() != .EndOfFile) { const entry_start = self.pos; if (self.peek() != .LowerIdent) { self.store.clearScratchRecordFieldsFrom(fields_scratch_top); @@ -534,18 +537,10 @@ pub fn parseStmt(self: *Parser) ?IR.NodeStore.StatementIdx { } }); return statement_idx; } else { - // If not a decl - const expr = self.parseExpr(); - const statement_idx = self.store.addStatement(.{ .expr = .{ - .expr = expr, - .region = .{ .start = start, .end = start }, - } }); - if (self.peek() == .Newline) { - self.advance(); - } - return statement_idx; + // continue to parse final expression } }, + // Expect to parse a Type Annotation, e.g. `Foo a : (a,a)` .UpperIdent => { const start = self.pos; if (self.peekNext() == .OpColon or self.peekNext() == .LowerIdent) { @@ -561,27 +556,24 @@ pub fn parseStmt(self: *Parser) ?IR.NodeStore.StatementIdx { .region = .{ .start = start, .end = self.pos }, } }); return statement_idx; + } else { + // continue to parse final expression } - const expr = self.parseExpr(); - const statement_idx = self.store.addStatement(.{ .expr = .{ - .expr = expr, - .region = .{ .start = start, .end = self.pos }, - } }); - return statement_idx; - }, - else => { - const start = self.pos; - const expr = self.parseExpr(); - const statement_idx = self.store.addStatement(.{ .expr = .{ - .expr = expr, - .region = .{ .start = start, .end = self.pos }, - } }); - if (self.peek() == .Newline) { - self.advance(); - } - return statement_idx; }, + else => {}, + } + + // We didn't find any statements, so we must be parsing the final expression. + const start = self.pos; + const expr = self.parseExpr(); + const statement_idx = self.store.addStatement(.{ .expr = .{ + .expr = expr, + .region = .{ .start = start, .end = self.pos }, + } }); + if (self.peek() == .Newline) { + self.advance(); } + return statement_idx; } /// Whether Pattern Alternatives are allowed in the current context @@ -627,7 +619,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt self.advance(); } self.store.clearScratchPatternsFrom(scratch_top); - return self.pushMalformed(IR.NodeStore.PatternIdx, .unexpected_token, start); + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, start); }; const args = self.store.patternSpanFrom(scratch_top); pattern = self.store.addPattern(.{ .tag = .{ @@ -665,7 +657,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt self.advance(); } self.store.clearScratchPatternsFrom(scratch_top); - return self.pushMalformed(IR.NodeStore.PatternIdx, .unexpected_token, start); + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, start); }; const patterns = self.store.patternSpanFrom(scratch_top); @@ -686,7 +678,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt } const fields = self.store.patternRecordFieldSpanFrom(scratch_top); if (self.peek() != .CloseCurly) { - return self.pushMalformed(IR.NodeStore.PatternIdx, .unexpected_token, start); + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, start); } self.advance(); pattern = self.store.addPattern(.{ .record = .{ @@ -701,7 +693,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt if (self.peek() == .KwAs) { self.advance(); if (self.peek() != .LowerIdent) { - return self.pushMalformed(IR.NodeStore.PatternIdx, .unexpected_token, start); + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, start); } name = self.pos; end = self.pos; @@ -726,7 +718,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt self.advance(); } self.store.clearScratchPatternsFrom(scratch_top); - return self.pushMalformed(IR.NodeStore.PatternIdx, .unexpected_token, start); + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, start); }; const patterns = self.store.patternSpanFrom(scratch_top); @@ -735,7 +727,9 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt .region = .{ .start = start, .end = self.pos }, } }); }, - else => std.debug.panic("TODO: Handle parsing pattern starting with: {s}", .{@tagName(self.peek())}), + else => { + return self.pushMalformed(IR.NodeStore.PatternIdx, .pattern_unexpected_token, self.pos); + }, } if (pattern) |p| { @@ -788,7 +782,7 @@ pub fn parsePatternRecordField(self: *Parser, alternatives: Alternatives) IR.Nod }); } if (self.peek() != .LowerIdent) { - while (self.peek() != .CloseCurly) { + while (self.peek() != .CloseCurly and self.peek() != .EndOfFile) { self.advance(); } return self.pushMalformed(IR.NodeStore.PatternRecordFieldIdx, .unexpected_token, field_start); @@ -797,7 +791,7 @@ pub fn parsePatternRecordField(self: *Parser, alternatives: Alternatives) IR.Nod self.advance(); var value: ?IR.NodeStore.PatternIdx = null; if (self.peek() != .OpColon and (self.peekNext() != .Comma or self.peekNext() != .CloseCurly)) { - while (self.peek() != .CloseCurly) { + while (self.peek() != .CloseCurly and self.peek() != .EndOfFile) { self.advance(); } return self.pushMalformed(IR.NodeStore.PatternRecordFieldIdx, .unexpected_token, field_start); @@ -874,7 +868,7 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { self.advance(); const scratch_top = self.store.scratchExprTop(); const list_end = self.parseCollectionSpan(IR.NodeStore.ExprIdx, .CloseSquare, IR.NodeStore.addScratchExpr, parseExpr) catch { - while (self.peek() != .CloseSquare) { + while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { self.advance(); } self.store.clearScratchExprsFrom(scratch_top); @@ -891,7 +885,7 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { // TODO: Parenthesized expressions const scratch_top = self.store.scratchExprTop(); _ = self.parseCollectionSpan(IR.NodeStore.ExprIdx, .CloseRound, IR.NodeStore.addScratchExpr, parseExpr) catch { - while (self.peek() != .CloseRound) { + while (self.peek() != .CloseRound and self.peek() != .EndOfFile) { self.advance(); } self.store.clearScratchExprsFrom(scratch_top); @@ -947,6 +941,7 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { return self.pushMalformed(IR.NodeStore.ExprIdx, .unexpected_token, start); }; const args = self.store.patternSpanFrom(scratch_top); + const body = self.parseExpr(); expr = self.store.addExpr(.{ .lambda = .{ .body = body, @@ -978,7 +973,7 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { return self.pushMalformed(IR.NodeStore.ExprIdx, .unexpected_token, start); }; const scratch_top = self.store.scratchWhenBranchTop(); - while (self.peek() != .CloseCurly) { + while (self.peek() != .CloseCurly and self.peek() != .EndOfFile) { self.store.addScratchWhenBranch(self.parseBranch()); if (self.peek() == .Comma) { self.advance(); @@ -1017,8 +1012,8 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { var expression = self.parseExprSuffix(start, e); while (self.peek() == .NoSpaceDotInt or self.peek() == .NoSpaceDotLowerIdent) { const tok = self.peek(); - if (tok == .NoSpaceDotInt) { // NoSpaceDotInt - std.debug.panic("TODO: Handle NoSpaceDotInt case", .{}); + if (tok == .NoSpaceDotInt) { + return self.pushMalformed(IR.NodeStore.ExprIdx, .expr_no_space_dot_int, self.pos); } else { // NoSpaceDotLowerIdent const s = self.pos; const ident = self.store.addExpr(.{ .ident = .{ @@ -1133,7 +1128,7 @@ pub fn parseStringExpr(self: *Parser) IR.NodeStore.ExprIdx { // StringStart, StringPart, OpenStringInterpolation, , CloseStringInterpolation, StringPart, StringEnd self.advanceOne(); const scratch_top = self.store.scratchExprTop(); - while (true) { + while (self.peek() != .EndOfFile) { switch (self.peek()) { .StringEnd => { self.advanceOne(); @@ -1158,8 +1153,7 @@ pub fn parseStringExpr(self: *Parser) IR.NodeStore.ExprIdx { }, else => { // Something is broken in the tokenizer if we get here! - std.debug.print("Unexpected token in string: {s}\n", .{@tagName(self.peek())}); - unreachable; + return self.pushMalformed(IR.NodeStore.ExprIdx, .string_unexpected_token, self.pos); }, } } @@ -1217,7 +1211,7 @@ const TyFnArgs = enum { looking_for_args, }; -/// todo +/// Parse a type annotation, e.g. `Foo(a) : (a,Str,I64)` pub fn parseTypeAnno(self: *Parser, looking_for_args: TyFnArgs) IR.NodeStore.TypeAnnoIdx { const start = self.pos; var anno: ?IR.NodeStore.TypeAnnoIdx = null; @@ -1262,7 +1256,7 @@ pub fn parseTypeAnno(self: *Parser, looking_for_args: TyFnArgs) IR.NodeStore.Typ self.advance(); // Advance past OpenRound const after_round = self.pos; const scratch_top = self.store.scratchTypeAnnoTop(); - while (self.peek() != .CloseRound and self.peek() != .OpArrow and self.peek() != .OpFatArrow) { + while (self.peek() != .CloseRound and self.peek() != .OpArrow and self.peek() != .OpFatArrow and self.peek() != .EndOfFile) { // Looking for args here so that we don't capture an un-parenthesized fn's args self.store.addScratchTypeAnno(self.parseTypeAnno(.looking_for_args)); if (self.peek() != .Comma) { @@ -1336,7 +1330,7 @@ pub fn parseTypeAnno(self: *Parser, looking_for_args: TyFnArgs) IR.NodeStore.Typ self.advance(); // Advance past Underscore }, else => { - std.debug.panic("Could not parse type annotation, got {s}@{d}", .{ @tagName(self.peek()), self.pos }); + return self.pushMalformed(IR.NodeStore.TypeAnnoIdx, .ty_anno_unexpected_token, self.pos); }, } @@ -1376,7 +1370,7 @@ pub fn parseTypeAnnoInCollection(self: *Parser) IR.NodeStore.TypeAnnoIdx { pub fn parseAnnoRecordField(self: *Parser) IR.NodeStore.AnnoRecordFieldIdx { const field_start = self.pos; if (self.peek() != .LowerIdent) { - while (self.peek() != .CloseCurly and self.peek() != .Comma) { + while (self.peek() != .CloseCurly and self.peek() != .Comma and self.peek() != .EndOfFile) { self.advance(); // Advance until we end this field or the record } return self.pushMalformed(IR.NodeStore.AnnoRecordFieldIdx, .unexpected_token, field_start); @@ -1384,7 +1378,7 @@ pub fn parseAnnoRecordField(self: *Parser) IR.NodeStore.AnnoRecordFieldIdx { const name = self.pos; self.advance(); // Advance past LowerIdent if (self.peek() != .OpColon) { - while (self.peek() != .CloseCurly and self.peek() != .Comma) { + while (self.peek() != .CloseCurly and self.peek() != .Comma and self.peek() != .EndOfFile) { self.advance(); // Advance until we end this field or the record } return self.pushMalformed(IR.NodeStore.AnnoRecordFieldIdx, .unexpected_token, field_start); diff --git a/src/check/parse/tokenize.zig b/src/check/parse/tokenize.zig index 10dc9a0d6a0..49ccffb9036 100644 --- a/src/check/parse/tokenize.zig +++ b/src/check/parse/tokenize.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Allocator = std.mem.Allocator; const collections = @import("../../collections.zig"); const exitOnOom = @import("../../collections/utils.zig").exitOnOom; const base = @import("../../base.zig"); @@ -328,8 +329,195 @@ pub const Diagnostic = struct { OverClosedBrace, MismatchedBrace, }; + + pub fn toStr(self: Diagnostic, gpa: Allocator, source: []const u8, writer: anytype) !void { + var newlines = std.ArrayList(usize).init(gpa); + defer newlines.deinit(); + + try newlines.append(0); + + // Find all newlines in the source + var pos: usize = 0; + for (source) |c| { + if (c == '\n') { + try newlines.append(pos + 1); // Position after the newline + } + pos += 1; + } + + // Get position information + const info = getDiagnosticPositionInfo(source, newlines, self.begin, self.end); + + // Strip trailing newline for display + const display_text = if (info.line_text.len > 0 and + (info.line_text[info.line_text.len - 1] == '\n' or + info.line_text[info.line_text.len - 1] == '\r')) + info.line_text[0 .. info.line_text.len - 1] + else + info.line_text; + + var spaces = std.ArrayList(u8).init(gpa); + defer spaces.deinit(); + for (0..info.start_col) |_| { + try spaces.append(' '); + } + + var carets = std.ArrayList(u8).init(gpa); + defer carets.deinit(); + + const caret_length = if (self.end > self.begin) self.end - self.begin else 1; + for (0..caret_length) |_| { + try carets.append('^'); + } + + const error_message = try std.fmt.allocPrint( + gpa, + "TOKENIZE: ({d}:{d}-{d}:{d}) {s}:\n{s}\n{s}{s}", + .{ info.start_line + 1, info.start_col + 1, info.end_line + 1, info.end_col + 1, @tagName(self.tag), display_text, spaces.items, carets.items }, + ); + defer gpa.free(error_message); + + try writer.writeAll(error_message); + } }; +/// Finds the line number for a given position in the source +fn lineNum(newlines: std.ArrayList(usize), pos: u32) u32 { + const pos_usize = @as(usize, @intCast(pos)); + + if (newlines.items.len == 0) { + return 0; + } + + var lineno: u32 = 0; + + while (lineno + 1 < newlines.items.len) { + if (newlines.items[lineno + 1] > pos_usize) { + return lineno; + } + lineno += 1; + } + + return lineno; +} + +/// Gets the column number for a position on a given line +fn columnNum(newlines: std.ArrayList(usize), line: u32, pos: u32) u32 { + const line_start: u32 = @intCast(newlines.items[line]); + return pos - line_start; +} + +/// Returns the source text for a given line +fn getLineText(source: []const u8, newlines: std.ArrayList(usize), line: u32) []const u8 { + const line_start = newlines.items[line]; + const line_end = if (line + 1 < newlines.items.len) + newlines.items[line + 1] + else + source.len; + + return source[line_start..line_end]; +} + +/// Returns the position info for a diagnostic +fn getDiagnosticPositionInfo(source: []const u8, newlines: std.ArrayList(usize), begin: u32, end: u32) struct { start_line: u32, start_col: u32, end_line: u32, end_col: u32, line_text: []const u8 } { + const start_line = lineNum(newlines, begin); + const start_col = columnNum(newlines, start_line, begin); + const end_line = lineNum(newlines, end); + const end_col = columnNum(newlines, end_line, end); + const line_text = getLineText(source, newlines, start_line); + + return .{ + .start_line = start_line, + .start_col = start_col, + .end_line = end_line, + .end_col = end_col, + .line_text = line_text, + }; +} + +test "lineNum" { + const gpa = std.testing.allocator; + var newlines = std.ArrayList(usize).init(gpa); + defer newlines.deinit(); + + // Simple test case with lines at positions 0, 10, 20 + try newlines.append(0); + try newlines.append(10); + try newlines.append(20); + try newlines.append(30); + + try std.testing.expectEqual(@as(u32, 0), lineNum(newlines, 0)); + try std.testing.expectEqual(@as(u32, 0), lineNum(newlines, 5)); + try std.testing.expectEqual(@as(u32, 0), lineNum(newlines, 9)); + try std.testing.expectEqual(@as(u32, 1), lineNum(newlines, 10)); + try std.testing.expectEqual(@as(u32, 1), lineNum(newlines, 15)); + try std.testing.expectEqual(@as(u32, 1), lineNum(newlines, 19)); + try std.testing.expectEqual(@as(u32, 2), lineNum(newlines, 20)); + try std.testing.expectEqual(@as(u32, 2), lineNum(newlines, 25)); + try std.testing.expectEqual(@as(u32, 2), lineNum(newlines, 29)); + try std.testing.expectEqual(@as(u32, 3), lineNum(newlines, 30)); + try std.testing.expectEqual(@as(u32, 3), lineNum(newlines, 35)); +} + +test "columnNum" { + const gpa = std.testing.allocator; + var newlines = std.ArrayList(usize).init(gpa); + defer newlines.deinit(); + + try newlines.append(0); + try newlines.append(10); + try newlines.append(20); + + try std.testing.expectEqual(@as(u32, 0), columnNum(newlines, 0, 0)); + try std.testing.expectEqual(@as(u32, 5), columnNum(newlines, 0, 5)); + try std.testing.expectEqual(@as(u32, 9), columnNum(newlines, 0, 9)); + + try std.testing.expectEqual(@as(u32, 0), columnNum(newlines, 1, 10)); + try std.testing.expectEqual(@as(u32, 5), columnNum(newlines, 1, 15)); +} + +test "getLineText" { + const gpa = std.testing.allocator; + var newlines = std.ArrayList(usize).init(gpa); + defer newlines.deinit(); + + const source = "line0\nline1\nline2"; + + try newlines.append(0); + try newlines.append(6); // After "line0\n" + try newlines.append(12); // After "line1\n" + + try std.testing.expectEqualStrings("line0\n", getLineText(source, newlines, 0)); + try std.testing.expectEqualStrings("line1\n", getLineText(source, newlines, 1)); + try std.testing.expectEqualStrings("line2", getLineText(source, newlines, 2)); +} + +test "getDiagnosticPositionInfo" { + const gpa = std.testing.allocator; + var newlines = std.ArrayList(usize).init(gpa); + defer newlines.deinit(); + + const source = "line0\nline1\nline2"; + + try newlines.append(0); + try newlines.append(6); // After "line0\n" + try newlines.append(12); // After "line1\n" + + const info1 = getDiagnosticPositionInfo(source, newlines, 2, 4); // "ne" in line0 + try std.testing.expectEqual(@as(u32, 0), info1.start_line); + try std.testing.expectEqual(@as(u32, 2), info1.start_col); + try std.testing.expectEqual(@as(u32, 0), info1.end_line); + try std.testing.expectEqual(@as(u32, 4), info1.end_col); + try std.testing.expectEqualStrings("line0\n", info1.line_text); + + const info2 = getDiagnosticPositionInfo(source, newlines, 8, 10); // "ne" in line1 + try std.testing.expectEqual(@as(u32, 1), info2.start_line); + try std.testing.expectEqual(@as(u32, 2), info2.start_col); + try std.testing.expectEqual(@as(u32, 1), info2.end_line); + try std.testing.expectEqual(@as(u32, 4), info2.end_col); + try std.testing.expectEqualStrings("line1\n", info2.line_text); +} + /// The cursor is our current position in the input text, and it collects messages. /// Note that instead of allocating its own message list, the caller must pass in a pre-allocated /// slice of Message. The field `message_count` tracks how many messages have been written. diff --git a/src/coordinate.zig b/src/coordinate.zig index 300a2b89c13..cf2a97579c9 100644 --- a/src/coordinate.zig +++ b/src/coordinate.zig @@ -462,6 +462,7 @@ fn findRootOfPackage( const ParsePackageDepsErr = union(enum) { failed_to_canonicalize_root_file: Filesystem.CanonicalizeError, failed_to_read_root_file: Filesystem.ReadError, + malformed_header, }; fn parseDependenciesFromPackageRoot( @@ -506,6 +507,9 @@ fn parseDependenciesFromPackageRoot( .start = 0, .len = 0, } }, + .malformed => { + return ParsePackageDepsErr.malformed_header; + }, }; for (parse_ast.store.recordFieldSlice(package_list)) |package_import| { diff --git a/src/fmt.zig b/src/fmt.zig index 1cdbf61f0b9..1d2206f7618 100644 --- a/src/fmt.zig +++ b/src/fmt.zig @@ -45,15 +45,28 @@ pub fn resetWith(fmt: *Formatter, ast: IR) void { /// Emits a string containing the well-formed source of a Roc parse IR (AST). /// The resulting string is owned by the caller. pub fn formatFile(fmt: *Formatter) []const u8 { + var ignore_newline_for_first_statement = false; + fmt.ast.store.emptyScratch(); const file = fmt.ast.store.getFile(); - fmt.formatHeader(file.header); + const maybe_output = fmt.formatHeader(file.header); + if (maybe_output == FormattedOutput.nothing_formatted) { + ignore_newline_for_first_statement = true; + } var newline_behavior: NewlineBehavior = .extra_newline_needed; for (fmt.ast.store.statementSlice(file.statements)) |s| { - fmt.ensureNewline(); - if (newline_behavior == .extra_newline_needed) { - fmt.newline(); + + // If there was nothing formatted because the header was malformed, + // then we don't want to add a newline + if (ignore_newline_for_first_statement) { + ignore_newline_for_first_statement = false; + } else { + fmt.ensureNewline(); + if (newline_behavior == .extra_newline_needed) { + fmt.newline(); + } } + newline_behavior = fmt.formatStatement(s); } return fmt.buffer.toOwnedSlice(fmt.gpa) catch |err| exitOnOom(err); @@ -122,6 +135,9 @@ fn formatStatement(fmt: *Formatter, si: StatementIdx) NewlineBehavior { fmt.formatExpr(r.expr); return .extra_newline_needed; }, + .malformed => { + return .no_extra_newline; + }, } } @@ -243,7 +259,18 @@ fn formatExpr(fmt: *Formatter, ei: ExprIdx) void { .lambda => |l| { fmt.push('|'); var i: usize = 0; - for (fmt.ast.store.patternSlice(l.args)) |arg| { + const arg_slice = fmt.ast.store.patternSlice(l.args); + + // TODO -- this is a hack to avoid ambiguity with no arguments, + // if we parse it again without the space it will be parsed as + // a logical OR `||` instead + // + // desired behaviour described here https://roc.zulipchat.com/#narrow/channel/395097-compiler-development/topic/zig.20compiler.20-.20spike/near/504453049 + if (arg_slice.len == 0) { + fmt.pushAll(" "); + } + + for (arg_slice) |arg| { fmt.formatPattern(arg); if (i < (l.args.span.len - 1)) { fmt.pushAll(", "); @@ -313,6 +340,9 @@ fn formatExpr(fmt: *Formatter, ei: ExprIdx) void { .ellipsis => |_| { fmt.pushAll("..."); }, + .malformed => { + // format nothing for malformed expressions + }, else => { std.debug.panic("TODO: Handle formatting {s}", .{@tagName(expr)}); }, @@ -414,6 +444,9 @@ fn formatPattern(fmt: *Formatter, pi: PatternIdx) void { i += 1; } }, + .malformed => { + // format nothing for malformed patterns + }, } } @@ -441,7 +474,10 @@ fn formatExposedItem(fmt: *Formatter, idx: IR.NodeStore.ExposedItemIdx) void { } } -fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { +/// The caller may need to know if anything was formatted, to handle newlines correctly. +const FormattedOutput = enum { something_formatted, nothing_formatted }; + +fn formatHeader(fmt: *Formatter, hi: HeaderIdx) FormattedOutput { const header = fmt.ast.store.getHeader(hi); switch (header) { .app => |a| { @@ -476,6 +512,7 @@ fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { } fmt.pushAll(" }"); fmt.newline(); + return FormattedOutput.something_formatted; }, .module => |m| { fmt.pushAll("module ["); @@ -489,6 +526,11 @@ fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { } fmt.push(']'); fmt.newline(); + return FormattedOutput.something_formatted; + }, + .malformed => { + // we have a malformed header... don't output anything as no header was parsed + return FormattedOutput.nothing_formatted; }, else => { std.debug.panic("TODO: Handle formatting {s}", .{@tagName(header)}); @@ -618,6 +660,9 @@ fn formatTypeAnno(fmt: *Formatter, anno: IR.NodeStore.TypeAnnoIdx) void { .underscore => |_| { fmt.push('_'); }, + .malformed => { + // format nothing for malformed type annotations + }, } } diff --git a/src/problem.zig b/src/problem.zig index 7a33406cab6..c3635bff2a3 100644 --- a/src/problem.zig +++ b/src/problem.zig @@ -6,6 +6,7 @@ //! our commitment to "always inform, never block" and to boost development speed. const std = @import("std"); +const Allocator = std.mem.Allocator; const base = @import("base.zig"); const collections = @import("collections.zig"); @@ -14,6 +15,8 @@ const Region = base.Region; /// Represents a problem encountered during the compilation process. pub const Problem = union(enum) { + tokenize: @import("check/parse/tokenize.zig").Diagnostic, + parser: @import("check/parse/IR.zig").Diagnostic, canonicalize: Canonicalize, compiler: Compiler, @@ -74,4 +77,27 @@ pub const Problem = union(enum) { pub const List = collections.SafeList(@This()); /// An index into a list of problems. pub const Idx = List.Idx; + + /// Format a `Problem` for display. + pub fn toStr(self: @This(), gpa: Allocator, source: []const u8, writer: anytype) !void { + + // use a stack allocation for printing our tag errors + var buf: [1000]u8 = undefined; + + switch (self) { + .tokenize => |a| try a.toStr(gpa, source, writer), + .parser => |a| { + const err_msg = try std.fmt.bufPrint(&buf, "PARSER: {s}", .{@tagName(a.tag)}); + try writer.writeAll(err_msg); + }, + .canonicalize => |err| { + const err_msg = try std.fmt.bufPrint(&buf, "CAN: {?}", .{err}); + try writer.writeAll(err_msg); + }, + .compiler => |err| { + const err_msg = try std.fmt.bufPrint(&buf, "COMPILER: {?}", .{err}); + try writer.writeAll(err_msg); + }, + } + } }; diff --git a/src/snapshot.zig b/src/snapshot.zig index 6a913857105..470a651eea8 100644 --- a/src/snapshot.zig +++ b/src/snapshot.zig @@ -149,18 +149,24 @@ const Section = union(enum) { source, formatted, parse, + tokens, + problems, pub const META = "~~~META"; pub const SOURCE = "~~~SOURCE"; pub const FORMATTED = "~~~FORMATTED"; pub const PARSE = "~~~PARSE"; + pub const TOKENS = "~~~TOKENS"; + pub const PROBLEMS = "~~~PROBLEMS"; fn next(self: Section) ?Section { return switch (self) { .meta => .source, - .source => .formatted, - .formatted => .parse, - .parse => null, + .source => .problems, + .problems => .formatted, + .formatted => .tokens, + .tokens => .parse, + .parse => .null, }; } @@ -169,6 +175,8 @@ const Section = union(enum) { if (std.mem.eql(u8, str, SOURCE)) return .source; if (std.mem.eql(u8, str, FORMATTED)) return .formatted; if (std.mem.eql(u8, str, PARSE)) return .parse; + if (std.mem.eql(u8, str, TOKENS)) return .tokens; + if (std.mem.eql(u8, str, PROBLEMS)) return .problems; return null; } @@ -178,6 +186,8 @@ const Section = union(enum) { .source => SOURCE, .formatted => FORMATTED, .parse => PARSE, + .tokens => TOKENS, + .problems => PROBLEMS, .None => "", }; } @@ -215,10 +225,6 @@ const Content = struct { }; } - fn has_formatted_section(self: Content) bool { - return self.formatted != null; - } - fn from_ranges(ranges: std.AutoHashMap(Section, Section.Range), content: []const u8) Error!Content { var meta: []const u8 = undefined; var source: []const u8 = undefined; @@ -291,12 +297,6 @@ fn processSnapshotFile(gpa: Allocator, snapshot_path: []const u8, maybe_fuzz_cor } }; - // std.debug.print("FILE: {s}\n{}\n", .{ snapshot_path, content }); - - // Generate the PARSE section - var parse_buffer = std.ArrayList(u8).init(gpa); - defer parse_buffer.deinit(); - var module_env = base.ModuleEnv.init(gpa); defer module_env.deinit(); @@ -304,52 +304,100 @@ fn processSnapshotFile(gpa: Allocator, snapshot_path: []const u8, maybe_fuzz_cor var parse_ast = parse.parse(&module_env, content.source); defer parse_ast.deinit(); - // Format the source code - var formatter = fmt.init(parse_ast); - defer formatter.deinit(); - const formatted_output = formatter.formatFile(); - defer gpa.free(formatted_output); - // shouldn't be required in future parse_ast.store.emptyScratch(); - // Write the new AST to the parse section - try parse_ast.toSExprStr(&module_env, parse_buffer.writer().any()); - - // Rewrite the file with updated sections + // Overwrite the snapshot file var file = std.fs.cwd().createFile(snapshot_path, .{}) catch |err| { log("failed to create file '{s}': {s}", .{ snapshot_path, @errorName(err) }); return false; }; defer file.close(); - try file.writer().writeAll(Section.META); - try file.writer().writeAll("\n"); - try file.writer().writeAll(content.meta); - try file.writer().writeAll("\n"); + // Copy original META + { + try file.writer().writeAll(Section.META); + try file.writer().writeAll("\n"); + try file.writer().writeAll(content.meta); + try file.writer().writeAll("\n"); + } - // If there's an explicit FORMATTED section, keep the source as-is - // and update the FORMATTED section - if (content.has_formatted_section()) { + // Copy original SOURCE + { try file.writer().writeAll(Section.SOURCE); try file.writer().writeAll("\n"); try file.writer().writeAll(content.source); try file.writer().writeAll("\n"); - try file.writer().writeAll(Section.FORMATTED); + } + + // Write out any PROBLEMS + { + try file.writer().writeAll(Section.PROBLEMS); try file.writer().writeAll("\n"); - try file.writer().writeAll(formatted_output); + if (module_env.problems.len() > 0) { + var iter = module_env.problems.iterIndices(); + while (iter.next()) |problem_idx| { + const problem = module_env.problems.get(problem_idx); + try problem.toStr(gpa, content.source, file); + try file.writer().writeAll("\n"); + } + } else { + try file.writer().writeAll("NIL\n"); + } + } + + // Write out any TOKENS + { + try file.writer().writeAll(Section.TOKENS); try file.writer().writeAll("\n"); - } else { - // Otherwise, update SOURCE directly with the formatted output - try file.writer().writeAll(Section.SOURCE); + const tokenizedBuffer = parse_ast.tokens; + const tokens = tokenizedBuffer.tokens.items(.tag); + var first = true; + for (tokens) |tok| { + + // only write a comma if not the first token + if (first) { + first = false; + } else { + try file.writer().writeAll(","); + } + + try file.writer().writeAll(@tagName(tok)); + } + try file.writer().writeAll("\n"); + } + + // Write PARSE SECTION + { + var parse_buffer = std.ArrayList(u8).init(gpa); + defer parse_buffer.deinit(); + try parse_ast.toSExprStr(&module_env, parse_buffer.writer().any()); + try file.writer().writeAll(Section.PARSE); try file.writer().writeAll("\n"); - try file.writer().writeAll(formatted_output); + try file.writer().writeAll(parse_buffer.items); try file.writer().writeAll("\n"); } - try file.writer().writeAll(Section.PARSE); - try file.writer().writeAll("\n"); - try file.writer().writeAll(parse_buffer.items); + // Write FORMAT SECTION + { + var formatter = fmt.init(parse_ast); + defer formatter.deinit(); + const formatted = formatter.formatFile(); + defer gpa.free(formatted); + + try file.writer().writeAll(Section.FORMATTED); + try file.writer().writeAll("\n"); + + if (!std.mem.eql(u8, formatted, content.source)) { + try file.writer().writeAll(formatted); + try file.writer().writeAll("\n"); + } else { + try file.writer().writeAll("NO CHANGE"); + try file.writer().writeAll("\n"); + } + } + + try file.writer().writeAll("~~~END"); // If flag --fuzz-corpus is passed, so write the SOURCE to our corpus if (maybe_fuzz_corpus_path != null) { @@ -384,6 +432,7 @@ fn processSnapshotFile(gpa: Allocator, snapshot_path: []const u8, maybe_fuzz_cor try corpus_file.writer().writeAll(content.source); } + // Log the file path that was written to log("{s}", .{snapshot_path}); return true; diff --git a/src/snapshots/001.txt b/src/snapshots/001.txt index 3785221d64b..6c93770b991 100644 --- a/src/snapshots/001.txt +++ b/src/snapshots/001.txt @@ -9,14 +9,19 @@ module [ foo = "one" +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,Newline,LowerIdent,Comma,Newline,CloseSquare,Newline,LowerIdent,OpAssign,Newline,StringStart,StringPart,StringEnd,EndOfFile +~~~PARSE +(file + (module + (exposed_item (lower_ident "foo"))) + (decl + (ident "foo") + (string "one"))) ~~~FORMATTED module [foo] foo = "one" -~~~PARSE -(file - (header - (exposed_item (lower_ident 'foo'))) - (decl - (ident 'foo') - (string 'one'))) \ No newline at end of file +~~~END \ No newline at end of file diff --git a/src/snapshots/003.txt b/src/snapshots/003.txt index f9faee8b946..e997c32c4df 100644 --- a/src/snapshots/003.txt +++ b/src/snapshots/003.txt @@ -12,21 +12,28 @@ crash "something" expect 1 == 1 return 2 +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,LowerIdent,CloseSquare,Newline,KwImport,LowerIdent,NoSpaceDotUpperIdent,Newline,LowerIdent,OpAssign,UpperIdent,NoSpaceDotLowerIdent,Newline,KwCrash,StringStart,StringPart,StringEnd,Newline,KwExpect,Int,OpEquals,Int,Newline,KwReturn,Int,EndOfFile ~~~PARSE (file - (header - (exposed_item (lower_ident 'decoder'))) + (module + (exposed_item (lower_ident "decoder"))) (import - 'json' - '.Json' - '') + "json" + ".Json" + "") (decl - (ident 'decoder') - (ident 'Utf8' '.decode')) - (crash (string 'something')) + (ident "decoder") + (ident "Utf8" ".decode")) + (crash (string "something")) (expect (binop - '==' - (int '1') - (int '1'))) - (return (int '2'))) \ No newline at end of file + "==" + (int "1") + (int "1"))) + (return (int "2"))) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file diff --git a/src/snapshots/add_var_with_spaces.txt b/src/snapshots/add_var_with_spaces.txt index 96e6d4d0231..f22c3f9eb96 100644 --- a/src/snapshots/add_var_with_spaces.txt +++ b/src/snapshots/add_var_with_spaces.txt @@ -4,17 +4,22 @@ description=Add a variable with spaces module [add2] add2 = x + 2 -~~~FORMATTED -module [add2] - -add2 = x + 2 +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,LowerIdent,CloseSquare,Newline,LowerIdent,OpAssign,LowerIdent,OpPlus,Int,EndOfFile ~~~PARSE (file - (header - (exposed_item (lower_ident 'add2'))) + (module + (exposed_item (lower_ident "add2"))) (decl - (ident 'add2') + (ident "add2") (binop - '+' - (ident '' 'x') - (int '2')))) \ No newline at end of file + "+" + (ident "" "x") + (int "2")))) +~~~FORMATTED +module [add2] + +add2 = x + 2 +~~~END \ No newline at end of file diff --git a/src/snapshots/expr_if_missing_else.txt b/src/snapshots/expr_if_missing_else.txt new file mode 100644 index 00000000000..9d2b8966140 --- /dev/null +++ b/src/snapshots/expr_if_missing_else.txt @@ -0,0 +1,21 @@ +~~~META +description= +~~~SOURCE +module [] + +foo = if tru then 0 +~~~PROBLEMS +PARSER: no_else +~~~TOKENS +KwModule,OpenSquare,CloseSquare,Newline,LowerIdent,OpAssign,KwIf,LowerIdent,LowerIdent,Int,EndOfFile +~~~PARSE +(file + (module) + (decl + (ident "foo") + (malformed_expr "no_else"))) +~~~FORMATTED +module [] + +foo = +~~~END \ No newline at end of file diff --git a/src/snapshots/expr_no_space_dot_int.txt b/src/snapshots/expr_no_space_dot_int.txt new file mode 100644 index 00000000000..e58581c397a --- /dev/null +++ b/src/snapshots/expr_no_space_dot_int.txt @@ -0,0 +1,21 @@ +~~~META +description= +~~~SOURCE +module [] + +foo = asd.0 +~~~PROBLEMS +PARSER: expr_no_space_dot_int +~~~TOKENS +KwModule,OpenSquare,CloseSquare,Newline,LowerIdent,OpAssign,LowerIdent,NoSpaceDotInt,EndOfFile +~~~PARSE +(file + (module) + (decl + (ident "foo") + (malformed_expr "expr_no_space_dot_int"))) +~~~FORMATTED +module [] + +foo = +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_001.txt b/src/snapshots/fuzz_crash_001.txt new file mode 100644 index 00000000000..1ad5fe3b949 --- /dev/null +++ b/src/snapshots/fuzz_crash_001.txt @@ -0,0 +1,17 @@ +~~~META +description=fuzz crash +~~~SOURCE +mo|% +~~~PROBLEMS +PARSER: missing_header +PARSER: pattern_unexpected_token +PARSER: unexpected_token +~~~TOKENS +LowerIdent,OpBar,OpPercent,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (malformed_expr "unexpected_token")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_002.txt b/src/snapshots/fuzz_crash_002.txt new file mode 100644 index 00000000000..66dca937efa --- /dev/null +++ b/src/snapshots/fuzz_crash_002.txt @@ -0,0 +1,68 @@ +~~~META +description=fuzz crash +~~~SOURCE +modu:;::::::::::::::le[% +~~~PROBLEMS +PARSER: missing_header +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +PARSER: unexpected_token +~~~TOKENS +LowerIdent,OpColon,MalformedUnknownToken,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,OpColon,LowerIdent,OpenSquare,OpPercent,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token") + (ident "" "le") + (malformed_expr "unexpected_token")) +~~~FORMATTED + + + + + + + + + + + + + + + + +le + + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_003.txt b/src/snapshots/fuzz_crash_003.txt new file mode 100644 index 00000000000..1fef23b26fa --- /dev/null +++ b/src/snapshots/fuzz_crash_003.txt @@ -0,0 +1,18 @@ +~~~META +description=fuzz crash +~~~SOURCE + = "te +~~~PROBLEMS +TOKENIZE: (1:5-1:7) UnclosedString: + = "te + ^^ +PARSER: missing_header +~~~TOKENS +OpAssign,StringStart,StringPart,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (string "te")) +~~~FORMATTED +"te" +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_004.txt b/src/snapshots/fuzz_crash_004.txt new file mode 100644 index 00000000000..22fabb1c136 --- /dev/null +++ b/src/snapshots/fuzz_crash_004.txt @@ -0,0 +1,13 @@ +~~~META +description=fuzz crash +~~~SOURCE +F +~~~PROBLEMS +PARSER: missing_header +~~~TOKENS +UpperIdent,EndOfFile +~~~PARSE +(file (malformed_header "missing_header")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_005.txt b/src/snapshots/fuzz_crash_005.txt new file mode 100644 index 00000000000..72e2fcdfcff --- /dev/null +++ b/src/snapshots/fuzz_crash_005.txt @@ -0,0 +1,13 @@ +~~~META +description=fuzz crash +~~~SOURCE +modu +~~~PROBLEMS +PARSER: missing_header +~~~TOKENS +LowerIdent,EndOfFile +~~~PARSE +(file (malformed_header "missing_header")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_006.txt b/src/snapshots/fuzz_crash_006.txt new file mode 100644 index 00000000000..4004ba99d0a Binary files /dev/null and b/src/snapshots/fuzz_crash_006.txt differ diff --git a/src/snapshots/fuzz_crash_007.txt b/src/snapshots/fuzz_crash_007.txt new file mode 100644 index 00000000000..9695a5f700d --- /dev/null +++ b/src/snapshots/fuzz_crash_007.txt @@ -0,0 +1,19 @@ +~~~META +description=fuzz crash +~~~SOURCE +ff8.8.d +~~~PROBLEMS +PARSER: missing_header +PARSER: unexpected_token +PARSER: unexpected_token +~~~TOKENS +LowerIdent,NoSpaceDotInt,NoSpaceDotLowerIdent,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (malformed_expr "unexpected_token") + (malformed_expr "unexpected_token")) +~~~FORMATTED + + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_008.txt b/src/snapshots/fuzz_crash_008.txt new file mode 100644 index 00000000000..255939190b9 --- /dev/null +++ b/src/snapshots/fuzz_crash_008.txt @@ -0,0 +1,19 @@ +~~~META +description=fuzz crash +~~~SOURCE +||1 +~~~PROBLEMS +TOKENIZE: (1:2-1:2) AsciiControl: +||1 + ^ +PARSER: missing_header +PARSER: unexpected_token +~~~TOKENS +OpBar,OpBar,Int,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (malformed_expr "unexpected_token")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_009.txt b/src/snapshots/fuzz_crash_009.txt new file mode 100644 index 00000000000..980674f0566 --- /dev/null +++ b/src/snapshots/fuzz_crash_009.txt @@ -0,0 +1,31 @@ +~~~META +description=fuzz crash +~~~SOURCE + f{o, + ] + +foo = + + "onmo % +~~~PROBLEMS +TOKENIZE: (2:6-2:6) MismatchedBrace: + ] + ^ +TOKENIZE: (6:6-6:12) UnclosedString: + "onmo % + ^^^^^^ +PARSER: missing_header +~~~TOKENS +LowerIdent,OpenCurly,LowerIdent,Comma,Newline,CloseCurly,Newline,LowerIdent,OpAssign,Newline,StringStart,StringPart,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (record (field "o")) + (decl + (ident "foo") + (string "onmo %"))) +~~~FORMATTED +{ o } + +foo = "onmo %" +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_010.txt b/src/snapshots/fuzz_crash_010.txt new file mode 100644 index 00000000000..cc9a9a6568d --- /dev/null +++ b/src/snapshots/fuzz_crash_010.txt @@ -0,0 +1,33 @@ +~~~META +description=fuzz crash +~~~SOURCE +H{o, +  ] +foo = + + "on (string 'onmo %'))) +~~~PROBLEMS +TOKENIZE: (2:3-2:3) AsciiControl: +  ] + ^ +TOKENIZE: (2:6-2:6) MismatchedBrace: +  ] + ^ +TOKENIZE: (5:6-5:35) UnclosedString: + "on (string 'onmo %'))) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +PARSER: missing_header +~~~TOKENS +UpperIdent,OpenCurly,LowerIdent,Comma,Newline,CloseCurly,Newline,LowerIdent,OpAssign,Newline,StringStart,StringPart,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (record (field "o")) + (decl + (ident "foo") + (string "on (string 'onmo %')))"))) +~~~FORMATTED +{ o } + +foo = "on (string 'onmo %')))" +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_011.txt b/src/snapshots/fuzz_crash_011.txt new file mode 100644 index 00000000000..8ac5068116d --- /dev/null +++ b/src/snapshots/fuzz_crash_011.txt @@ -0,0 +1,18 @@ +~~~META +description=fuzz crash +~~~SOURCE +module P]F +~~~PROBLEMS +TOKENIZE: (1:9-1:9) OverClosedBrace: +module P]F + ^ +PARSER: header_expected_open_square +~~~TOKENS +KwModule,UpperIdent,UpperIdent,EndOfFile +~~~PARSE +(file + (malformed_header "header_expected_open_square") + (tag "F")) +~~~FORMATTED +F +~~~END \ No newline at end of file diff --git a/src/snapshots/fuzz_crash_012.txt b/src/snapshots/fuzz_crash_012.txt new file mode 100644 index 00000000000..a13f3cbbca4 --- /dev/null +++ b/src/snapshots/fuzz_crash_012.txt @@ -0,0 +1,18 @@ +~~~META +description=fuzz crash +~~~SOURCE + ||(|(l888888888| +~~~PROBLEMS +PARSER: missing_header +PARSER: pattern_unexpected_token +PARSER: pattern_unexpected_token +PARSER: unexpected_token +~~~TOKENS +OpBar,OpBar,NoSpaceOpenRound,OpBar,NoSpaceOpenRound,LowerIdent,OpBar,EndOfFile +~~~PARSE +(file + (malformed_header "missing_header") + (malformed_expr "unexpected_token")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/header_expected_open_bracket.txt b/src/snapshots/header_expected_open_bracket.txt new file mode 100644 index 00000000000..69bd3776ef1 --- /dev/null +++ b/src/snapshots/header_expected_open_bracket.txt @@ -0,0 +1,13 @@ +~~~META +description= +~~~SOURCE +module +~~~PROBLEMS +PARSER: header_expected_open_square +~~~TOKENS +KwModule,EndOfFile +~~~PARSE +(file (malformed_header "header_expected_open_square")) +~~~FORMATTED + +~~~END \ No newline at end of file diff --git a/src/snapshots/hello_world.txt b/src/snapshots/hello_world.txt new file mode 100644 index 00000000000..997b02476d6 --- /dev/null +++ b/src/snapshots/hello_world.txt @@ -0,0 +1,29 @@ +~~~META +description=Hello world +~~~SOURCE +app [main!] { pf: platform "../basic-cli/platform.roc" } + +import pf.Stdout + +main! = |_| Stdout.line!("Hello, world!") +~~~PROBLEMS +NIL +~~~TOKENS +KwApp,OpenSquare,LowerIdent,CloseSquare,OpenCurly,LowerIdent,OpColon,KwPlatform,StringStart,StringPart,StringEnd,CloseCurly,Newline,KwImport,LowerIdent,NoSpaceDotUpperIdent,Newline,LowerIdent,OpAssign,OpBar,Underscore,OpBar,UpperIdent,NoSpaceDotLowerIdent,NoSpaceOpenRound,StringStart,StringPart,StringEnd,CloseRound,EndOfFile +~~~PARSE +(file + (app "TODO implement toSExpr for app module header") + (import + "pf" + ".Stdout" + "") + (decl + (ident "main!") + (lambda + (args (underscore)) + (apply + (ident "Stdout" ".line!") + (string "Hello, world!"))))) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file diff --git a/src/snapshots/if_then_else.txt b/src/snapshots/if_then_else.txt index 1b22c272cd7..b27541ac841 100644 --- a/src/snapshots/if_then_else.txt +++ b/src/snapshots/if_then_else.txt @@ -8,17 +8,22 @@ foo = if true A else { B } -~~~FORMATTED -module [foo] - -foo = if true A else B +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,LowerIdent,CloseSquare,Newline,LowerIdent,OpAssign,KwIf,LowerIdent,UpperIdent,Newline,KwElse,OpenCurly,Newline,UpperIdent,Newline,CloseCurly,EndOfFile ~~~PARSE (file - (header - (exposed_item (lower_ident 'foo'))) + (module + (exposed_item (lower_ident "foo"))) (decl - (ident 'foo') + (ident "foo") (if_then_else - (ident '' 'true') - (tag 'A') - (block (tag 'B'))))) \ No newline at end of file + (ident "" "true") + (tag "A") + (block (tag "B"))))) +~~~FORMATTED +module [foo] + +foo = if true A else B +~~~END \ No newline at end of file diff --git a/src/snapshots/some_folder/002.txt b/src/snapshots/some_folder/002.txt index 777310f8448..b6e45a4ad30 100644 --- a/src/snapshots/some_folder/002.txt +++ b/src/snapshots/some_folder/002.txt @@ -6,14 +6,21 @@ module [foo, bar] foo = "one" bar = "two" +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,LowerIdent,Comma,LowerIdent,CloseSquare,Newline,LowerIdent,OpAssign,StringStart,StringPart,StringEnd,Newline,LowerIdent,OpAssign,StringStart,StringPart,StringEnd,EndOfFile ~~~PARSE (file - (header - (exposed_item (lower_ident 'foo')) - (exposed_item (lower_ident 'bar'))) + (module + (exposed_item (lower_ident "foo")) + (exposed_item (lower_ident "bar"))) (decl - (ident 'foo') - (string 'one')) + (ident "foo") + (string "one")) (decl - (ident 'bar') - (string 'two'))) \ No newline at end of file + (ident "bar") + (string "two"))) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file diff --git a/src/snapshots/string.txt b/src/snapshots/string.txt new file mode 100644 index 00000000000..1b3e19cb787 --- /dev/null +++ b/src/snapshots/string.txt @@ -0,0 +1,20 @@ +~~~META +description=two strings +~~~SOURCE +module [] + +"one" + +"two" +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,CloseSquare,Newline,StringStart,StringPart,StringEnd,Newline,StringStart,StringPart,StringEnd,EndOfFile +~~~PARSE +(file + (module) + (string "one") + (string "two")) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file diff --git a/src/snapshots/type_annotations.txt b/src/snapshots/type_annotations.txt index edbdfbcfe21..0e046c0c11e 100644 --- a/src/snapshots/type_annotations.txt +++ b/src/snapshots/type_annotations.txt @@ -8,38 +8,45 @@ bar : Thing(a, b, _) baz : (a, b, c) add_one : (U8, U16 -> U32) main! : List(String) -> Result({}, _) +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,CloseSquare,Newline,LowerIdent,OpColon,UpperIdent,Newline,LowerIdent,OpColon,UpperIdent,NoSpaceOpenRound,LowerIdent,Comma,LowerIdent,Comma,Underscore,CloseRound,Newline,LowerIdent,OpColon,OpenRound,LowerIdent,Comma,LowerIdent,Comma,LowerIdent,CloseRound,Newline,LowerIdent,OpColon,OpenRound,UpperIdent,Comma,UpperIdent,OpArrow,UpperIdent,CloseRound,Newline,LowerIdent,OpColon,UpperIdent,NoSpaceOpenRound,UpperIdent,CloseRound,OpArrow,UpperIdent,NoSpaceOpenRound,OpenCurly,CloseCurly,Comma,Underscore,CloseRound,EndOfFile ~~~PARSE (file - (header) + (module) (type_anno - 'foo' - (tag 'U64')) + "foo" + (tag "U64")) (type_anno - 'bar' + "bar" (tag - 'Thing' - (ty_var 'a') - (ty_var 'b') + "Thing" + (ty_var "a") + (ty_var "b") (_))) (type_anno - 'baz' + "baz" (tuple - (ty_var 'a') - (ty_var 'b') - (ty_var 'c'))) + (ty_var "a") + (ty_var "b") + (ty_var "c"))) (type_anno - 'add_one' + "add_one" (fn - (tag 'U32') - (tag 'U8') - (tag 'U16'))) + (tag "U32") + (tag "U8") + (tag "U16"))) (type_anno - 'main!' + "main!" (fn (tag - 'Result' + "Result" (record) (_)) (tag - 'List' - (tag 'String'))))) \ No newline at end of file + "List" + (tag "String"))))) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file diff --git a/src/snapshots/type_declarations.txt b/src/snapshots/type_declarations.txt index e44bfd40d72..96dd06837a1 100644 --- a/src/snapshots/type_declarations.txt +++ b/src/snapshots/type_declarations.txt @@ -12,49 +12,56 @@ Some a : { foo : Ok(a), bar : Something } Maybe a : [Some(a), None] SomeFunc a : Maybe(a), a -> Maybe(a) +~~~PROBLEMS +NIL +~~~TOKENS +KwModule,OpenSquare,UpperIdent,Comma,UpperIdent,Comma,UpperIdent,Comma,UpperIdent,Comma,UpperIdent,Comma,LowerIdent,Comma,LowerIdent,CloseSquare,Newline,UpperIdent,LowerIdent,LowerIdent,OpColon,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,Comma,OpenRound,LowerIdent,OpArrow,LowerIdent,CloseRound,OpArrow,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,Newline,UpperIdent,OpColon,OpenRound,UpperIdent,Comma,UpperIdent,CloseRound,Newline,UpperIdent,LowerIdent,OpColon,OpenCurly,LowerIdent,OpColon,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,Comma,LowerIdent,OpColon,UpperIdent,CloseCurly,Newline,UpperIdent,LowerIdent,OpColon,OpenSquare,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,Comma,UpperIdent,CloseSquare,Newline,UpperIdent,LowerIdent,OpColon,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,Comma,LowerIdent,OpArrow,UpperIdent,NoSpaceOpenRound,LowerIdent,CloseRound,EndOfFile ~~~PARSE (file - (header - (exposed_item (upper_ident 'Map')) - (exposed_item (upper_ident 'Foo')) - (exposed_item (upper_ident 'Some')) - (exposed_item (upper_ident 'Maybe')) - (exposed_item (upper_ident 'SomeFunc')) - (exposed_item (lower_ident 'add_one')) - (exposed_item (lower_ident 'main!'))) + (module + (exposed_item (upper_ident "Map")) + (exposed_item (upper_ident "Foo")) + (exposed_item (upper_ident "Some")) + (exposed_item (upper_ident "Maybe")) + (exposed_item (upper_ident "SomeFunc")) + (exposed_item (lower_ident "add_one")) + (exposed_item (lower_ident "main!"))) (type_decl (header - 'Map' - 'a' - 'b') + "Map" + "a" + "b") (fn (tag - 'List' - (ty_var 'b')) + "List" + (ty_var "b")) (tag - 'List' - (ty_var 'a')) + "List" + (ty_var "a")) (fn - (ty_var 'b') - (ty_var 'a')))) + (ty_var "b") + (ty_var "a")))) (type_decl - (header 'Foo') + (header "Foo") (tuple - (tag 'Bar') - (tag 'Baz'))) + (tag "Bar") + (tag "Baz"))) (type_decl - (header 'Some' 'a') - (record '' '')) + (header "Some" "a") + (record "" "")) (type_decl - (header 'Maybe' 'a') - (tag_union 'TODO tags' 'TODO open_anno')) + (header "Maybe" "a") + (tag_union "TODO tags" "TODO open_anno")) (type_decl - (header 'SomeFunc' 'a') + (header "SomeFunc" "a") (fn (tag - 'Maybe' - (ty_var 'a')) + "Maybe" + (ty_var "a")) (tag - 'Maybe' - (ty_var 'a')) - (ty_var 'a')))) \ No newline at end of file + "Maybe" + (ty_var "a")) + (ty_var "a")))) +~~~FORMATTED +NO CHANGE +~~~END \ No newline at end of file