From edad3c728fcd401ffc91ad5455c77aa8eee11bec Mon Sep 17 00:00:00 2001 From: Anthony Bullard Date: Fri, 7 Mar 2025 09:57:22 -0600 Subject: [PATCH 1/5] [zig] Parser: Exposed items for imports/headers --- src/check/parse.zig | 8 +- src/check/parse/IR.zig | 242 ++++++++++++++++++++++++++++++++--- src/check/parse/Parser.zig | 196 +++++++++++++++++++--------- src/check/parse/tokenize.zig | 15 +++ src/fmt.zig | 57 ++++++++- 5 files changed, 432 insertions(+), 86 deletions(-) diff --git a/src/check/parse.zig b/src/check/parse.zig index 2703e2f3de..e98032d321 100644 --- a/src/check/parse.zig +++ b/src/check/parse.zig @@ -100,6 +100,10 @@ fn testSExprHelper(source: []const u8, expected: []const u8) !void { // parse our source var parse_ast = parse(&env, source); defer parse_ast.deinit(); + std.testing.expectEqualSlices(IR.Diagnostic, &[_]IR.Diagnostic{}, parse_ast.errors) catch { + std.debug.print("Tokens:\n{any}", .{parse_ast.tokens.tokens.items(.tag)}); + std.debug.panic("Test failed with parse errors", .{}); + }; // shouldn't be required in future parse_ast.store.emptyScratch(); @@ -126,7 +130,9 @@ test "example s-expr" { const expected = \\(file - \\ (header 'foo' 'bar') + \\ (header + \\ (exposed_item (lower_ident 'foo')) + \\ (exposed_item (lower_ident 'bar'))) \\ (decl \\ (ident 'foo') \\ (string 'hey')) diff --git a/src/check/parse/IR.zig b/src/check/parse/IR.zig index 83ef3d0a98..9885eeea13 100644 --- a/src/check/parse/IR.zig +++ b/src/check/parse/IR.zig @@ -48,6 +48,9 @@ pub const Diagnostic = struct { expected_platform_string, expected_package_or_platform_string, expected_package_platform_close_curly, + import_exposing_no_open, + import_exposing_no_close, + no_else, }; }; @@ -165,6 +168,11 @@ pub const Node = struct { /// * extra_data format: [[where node index]?, type_term node index] type_anno, + // Exposed items + exposed_item_lower, + exposed_item_upper, + exposed_item_upper_star, + // Type terms /// DESCRIPTION @@ -431,6 +439,7 @@ pub const NodeStore = struct { scratch_when_branches: std.ArrayListUnmanaged(WhenBranchIdx), scratch_type_annos: std.ArrayListUnmanaged(TypeAnnoIdx), scratch_anno_record_fields: std.ArrayListUnmanaged(AnnoRecordFieldIdx), + scratch_exposed_items: std.ArrayListUnmanaged(ExposedItemIdx), /// Initialize the store with an assumed capacity to /// ensure resizing of underlying data structures happens @@ -449,6 +458,7 @@ pub const NodeStore = struct { .scratch_when_branches = .{}, .scratch_type_annos = .{}, .scratch_anno_record_fields = .{}, + .scratch_exposed_items = .{}, }; store.nodes.ensureTotalCapacity(gpa, capacity); @@ -467,6 +477,7 @@ pub const NodeStore = struct { store.scratch_when_branches.ensureTotalCapacity(gpa, scratch_90th_percentile_capacity) catch |err| exitOnOom(err); store.scratch_type_annos.ensureTotalCapacity(gpa, scratch_90th_percentile_capacity) catch |err| exitOnOom(err); store.scratch_anno_record_fields.ensureTotalCapacity(gpa, scratch_90th_percentile_capacity) catch |err| exitOnOom(err); + store.scratch_exposed_items.ensureTotalCapacity(gpa, scratch_90th_percentile_capacity) catch |err| exitOnOom(err); return store; } @@ -494,6 +505,7 @@ pub const NodeStore = struct { store.scratch_when_branches.deinit(store.gpa); store.scratch_type_annos.deinit(store.gpa); store.scratch_anno_record_fields.deinit(store.gpa); + store.scratch_exposed_items.deinit(store.gpa); } /// Ensures that all scratch buffers in the store @@ -508,6 +520,7 @@ pub const NodeStore = struct { store.scratch_when_branches.shrinkRetainingCapacity(0); store.scratch_type_annos.shrinkRetainingCapacity(0); store.scratch_anno_record_fields.shrinkRetainingCapacity(0); + store.scratch_exposed_items.shrinkRetainingCapacity(0); } // Node Type Idx types @@ -516,6 +529,8 @@ pub const NodeStore = struct { pub const BodyIdx = struct { id: u32 }; /// An index for a Header node. Should not be constructed externally. pub const HeaderIdx = struct { id: u32 }; + /// An index for a ExposedItem node. Should not be constructed externally. + pub const ExposedItemIdx = struct { id: u32 }; /// An index for a Statement node. Should not be constructed externally. pub const StatementIdx = struct { id: u32 }; /// An index for a Pattern node. Should not be constructed externally. @@ -583,7 +598,7 @@ pub const NodeStore = struct { // } node.tag = .app_header; node.main_token = app.platform_name; - node.data.lhs = app.packages.span.start; + node.data.lhs = app.provides.span.start; node.data.rhs = @as(u32, @bitCast(Header.AppHeaderRhs{ .num_packages = @as(u10, @intCast(app.packages.span.len)), .num_provides = @as(u22, @intCast(app.provides.span.len)), @@ -602,6 +617,45 @@ pub const NodeStore = struct { return .{ .id = @intFromEnum(nid) }; } + pub fn addExposedItem(store: *NodeStore, item: ExposedItem) ExposedItemIdx { + var node = Node{ + .tag = .malformed, + .main_token = 0, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }; + + switch (item) { + .lower_ident => |i| { + node.tag = .exposed_item_lower; + node.main_token = i.ident; + if (i.as) |a| { + std.debug.assert(a > 0); + node.data.lhs = a; + node.data.rhs = 1; + } + }, + .upper_ident => |i| { + node.tag = .exposed_item_upper; + node.main_token = i.ident; + if (i.as) |a| { + std.debug.assert(a > 0); + node.data.lhs = a; + node.data.rhs = 1; + } + }, + .upper_ident_star => |i| { + node.tag = .exposed_item_upper_star; + node.main_token = i.ident; + }, + } + + const nid = store.nodes.append(store.gpa, node); + return .{ .id = @intFromEnum(nid) }; + } + pub fn addStatement(store: *NodeStore, statement: Statement) StatementIdx { var node = Node{ .tag = .statement, @@ -641,18 +695,25 @@ pub const NodeStore = struct { .qualified = 0, .num_exposes = @as(u30, @intCast(i.exposes.span.len)), }; - const extra_data_start = store.extra_data.items.len; + var ed_start: u32 = i.exposes.span.start; if (i.qualifier_tok) |tok| { rhs.qualified = 1; + if (ed_start == 0) { + ed_start = @intCast(store.extra_data.items.len); + } + store.extra_data.append(store.gpa, tok) catch |err| exitOnOom(err); } if (i.alias_tok) |tok| { rhs.aliased = 1; + if (ed_start == 0) { + ed_start = @intCast(store.extra_data.items.len); + } store.extra_data.append(store.gpa, tok) catch |err| exitOnOom(err); } node.data.rhs = @as(u32, @bitCast(rhs)); if (node.data.rhs > 0) { - node.data.lhs = @as(u32, @intCast(extra_data_start)); + node.data.lhs = ed_start; } }, .type_decl => |d| { @@ -1037,18 +1098,21 @@ pub const NodeStore = struct { .app_header => { const extra_data_start = node.data.lhs; const rhs = @as(Header.AppHeaderRhs, @bitCast(node.data.rhs)); - const platform_idx = @as(usize, @intCast(extra_data_start + rhs.num_packages + rhs.num_provides)) - 1; + var platform_idx = @as(usize, @intCast(extra_data_start + rhs.num_packages + rhs.num_provides)) + 1; + if (platform_idx < extra_data_start) { + platform_idx = @intCast(extra_data_start); + } const platform = store.extra_data.items[platform_idx]; return .{ .app = .{ .platform = .{ .id = platform }, .platform_name = node.main_token, .packages = .{ .span = .{ - .start = extra_data_start, + .start = extra_data_start + rhs.num_provides, .len = rhs.num_packages, } }, .provides = .{ .span = .{ - .start = extra_data_start + rhs.num_packages, + .start = extra_data_start, .len = rhs.num_provides, } }, .region = emptyRegion(), @@ -1070,6 +1134,49 @@ pub const NodeStore = struct { } } + pub fn getExposedItem(store: *NodeStore, item: ExposedItemIdx) ExposedItem { + const node = store.nodes.get(@enumFromInt(item.id)); + switch (node.tag) { + .exposed_item_lower => { + if (node.data.rhs == 1) { + return .{ .lower_ident = .{ + .region = emptyRegion(), + .ident = node.main_token, + .as = node.data.lhs, + } }; + } + return .{ .lower_ident = .{ + .region = emptyRegion(), + .ident = node.main_token, + .as = null, + } }; + }, + .exposed_item_upper => { + if (node.data.rhs == 1) { + return .{ .upper_ident = .{ + .region = emptyRegion(), + .ident = node.main_token, + .as = node.data.lhs, + } }; + } + return .{ .upper_ident = .{ + .region = emptyRegion(), + .ident = node.main_token, + .as = null, + } }; + }, + .exposed_item_upper_star => { + return .{ .upper_ident_star = .{ + .region = emptyRegion(), + .ident = node.main_token, + } }; + }, + else => { + std.debug.panic("Expected a valid exposed item tag, got {s}", .{@tagName(node.tag)}); + }, + } + } + pub fn emptyRegion() Region { return .{ .start = 0, .end = 0 }; } @@ -1092,7 +1199,7 @@ pub const NodeStore = struct { }, .import => { const rhs = @as(ImportRhs, @bitCast(node.data.rhs)); - var extra_data_pos = node.data.lhs; + var extra_data_pos = node.data.lhs + rhs.num_exposes; var qualifier_tok: ?TokenIdx = null; var alias_tok: ?TokenIdx = null; if (rhs.qualified == 1) { @@ -1101,18 +1208,19 @@ pub const NodeStore = struct { } if (rhs.aliased == 1) { alias_tok = store.extra_data.items[extra_data_pos]; - extra_data_pos += 1; } - return .{ .import = .{ + const i = Statement.Import{ .module_name_tok = node.main_token, .qualifier_tok = qualifier_tok, .alias_tok = alias_tok, .exposes = .{ .span = .{ - .start = extra_data_pos, + .start = node.data.lhs, .len = rhs.num_exposes, } }, .region = emptyRegion(), - } }; + }; + const imp = Statement{ .import = i }; + return imp; }, .expect => { return .{ .expect = .{ @@ -1576,18 +1684,18 @@ pub const NodeStore = struct { /// Represents a module header. pub const Header = union(enum) { app: struct { - provides: TokenSpan, // This should probably be a Interned Ident token + provides: ExposedItemSpan, // This should probably be a Interned Ident token platform: ExprIdx, platform_name: TokenIdx, packages: RecordFieldSpan, region: Region, }, module: struct { - exposes: TokenSpan, + exposes: ExposedItemSpan, region: Region, }, package: struct { - provides: TokenSpan, + provides: ExposedItemSpan, packages: RecordFieldSpan, region: Region, }, @@ -1607,8 +1715,10 @@ pub const NodeStore = struct { .module => |module| { var header_node = sexpr.Expr.init(env.gpa, "header"); - for (ir.store.tokenSlice(module.exposes)) |exposed_idx| { - header_node.appendStringChild(env.gpa, ir.resolve(exposed_idx)); + for (ir.store.exposedItemSlice(module.exposes)) |exposed| { + const item = ir.store.getExposedItem(exposed); + var item_node = item.toSExpr(env, ir); + header_node.appendNodeChild(env.gpa, &item_node); } return header_node; @@ -1618,6 +1728,57 @@ pub const NodeStore = struct { } }; + pub const ExposedItem = union(enum) { + lower_ident: struct { + as: ?TokenIdx, + ident: TokenIdx, + region: Region, + }, + upper_ident: struct { + as: ?TokenIdx, + ident: TokenIdx, + region: Region, + }, + upper_ident_star: struct { + ident: TokenIdx, + region: Region, + }, + + pub fn toSExpr(self: @This(), env: *base.ModuleEnv, ir: *IR) sexpr.Expr { + var node = sexpr.Expr.init(env.gpa, "exposed_item"); + var inner_node = sexpr.Expr.init(env.gpa, @tagName(self)); + switch (self) { + .lower_ident => |i| { + const token = ir.tokens.tokens.get(i.ident); + const text = env.idents.getText(token.extra.interned); + inner_node.appendStringChild(env.gpa, text); + if (i.as) |a| { + const as_tok = ir.tokens.tokens.get(a); + const as_text = env.idents.getText(as_tok.extra.interned); + inner_node.appendStringChild(env.gpa, as_text); + } + }, + .upper_ident => |i| { + const token = ir.tokens.tokens.get(i.ident); + const text = env.idents.getText(token.extra.interned); + inner_node.appendStringChild(env.gpa, text); + if (i.as) |a| { + const as_tok = ir.tokens.tokens.get(a); + const as_text = env.idents.getText(as_tok.extra.interned); + inner_node.appendStringChild(env.gpa, as_text); + } + }, + .upper_ident_star => |i| { + const token = ir.tokens.tokens.get(i.ident); + const text = env.idents.getText(token.extra.interned); + inner_node.appendStringChild(env.gpa, text); + }, + } + node.appendNodeChild(env.gpa, &inner_node); + return node; + } + }; + /// Represents a statement. Not all statements are valid in all positions. pub const Statement = union(enum) { decl: struct { @@ -1657,7 +1818,7 @@ pub const NodeStore = struct { module_name_tok: TokenIdx, qualifier_tok: ?TokenIdx, alias_tok: ?TokenIdx, - exposes: TokenSpan, + exposes: ExposedItemSpan, region: Region, }; @@ -1702,11 +1863,13 @@ pub const NodeStore = struct { ); // Each exposed identifier e.g. [foo, bar] in `import pf.Stdout exposing [foo, bar]` - const exposed_slice = ir.store.tokenSlice(import.exposes); + const exposed_slice = ir.store.exposedItemSlice(import.exposes); if (exposed_slice.len > 0) { var exposed = sexpr.Expr.init(env.gpa, "exposing"); - for (ir.store.tokenSlice(import.exposes)) |tok| { - exposed.appendStringChild(env.gpa, ir.resolve(tok)); + for (ir.store.exposedItemSlice(import.exposes)) |e| { + var exposed_item = &ir.store.getExposedItem(e); + var exposed_item_sexpr = exposed_item.toSExpr(env, ir); + exposed.appendNodeChild(env.gpa, &exposed_item_sexpr); } node.appendNodeChild(env.gpa, &exposed); } @@ -2162,6 +2325,7 @@ pub const NodeStore = struct { pub const WhenBranchSpan = struct { span: DataSpan }; pub const TypeAnnoSpan = struct { span: DataSpan }; pub const AnnoRecordFieldSpan = struct { span: DataSpan }; + pub const ExposedItemSpan = struct { span: DataSpan }; /// Returns the start position for a new Span of ExprIdxs in scratch pub fn scratchExprTop(store: *NodeStore) u32 { @@ -2496,6 +2660,44 @@ pub const NodeStore = struct { pub fn tokenSlice(store: *NodeStore, span: TokenSpan) []TokenIdx { return @ptrCast(store.extra_data.items[span.span.start..(span.span.start + span.span.len)]); } + + /// Returns the start position for a new Span of exposedItemIdxs in scratch + pub fn scratchExposedItemTop(store: *NodeStore) u32 { + return @as(u32, @intCast(store.scratch_anno_record_fields.items.len)); + } + + /// Places a new ExposedItemIdx in the scratch. Will panic on OOM. + pub fn addScratchExposedItem(store: *NodeStore, idx: ExposedItemIdx) void { + store.scratch_exposed_items.append(store.gpa, idx) catch |err| exitOnOom(err); + } + + /// Creates a new span starting at start. Moves the items from scratch + /// to extra_data as appropriate. + pub fn exposedItemSpanFrom(store: *NodeStore, start: u32) ExposedItemSpan { + const end = store.scratch_exposed_items.items.len; + defer store.scratch_exposed_items.shrinkRetainingCapacity(start); + var i = @as(usize, @intCast(start)); + const ed_start = @as(u32, @intCast(store.extra_data.items.len)); + while (i < end) { + store.extra_data.append(store.gpa, store.scratch_exposed_items.items[i].id) catch |err| exitOnOom(err); + i += 1; + } + const span = DataSpan{ .start = ed_start, .len = @as(u32, @intCast(end)) - start }; + return .{ .span = span }; + } + + /// Clears any ExposedItemIds added to scratch from start until the end. + /// Should be used wherever the scratch items will not be used, + /// as in when parsing fails. + pub fn clearScratchExposedItemsFrom(store: *NodeStore, start: u32) void { + store.scratch_exposed_items.shrinkRetainingCapacity(start); + } + + /// Returns a new ExposedItem slice so that the caller can iterate through + /// all items in the span. + pub fn exposedItemSlice(store: *NodeStore, span: ExposedItemSpan) []ExposedItemIdx { + return @ptrCast(store.extra_data.items[span.span.start..(span.span.start + span.span.len)]); + } }; /// Resolve a token index to a string slice from the source code. diff --git a/src/check/parse/Parser.zig b/src/check/parse/Parser.zig index 1efe44ea09..de24d77308 100644 --- a/src/check/parse/Parser.zig +++ b/src/check/parse/Parser.zig @@ -116,6 +116,7 @@ pub fn pushDiagnostic(self: *Parser, tag: IR.Diagnostic.Tag, region: IR.Region) } /// add a malformed token pub fn pushMalformed(self: *Parser, comptime t: type, tag: IR.Diagnostic.Tag) t { + std.debug.panic("!!!! pushMalformed t={any} tag={s}\n", .{ t, @tagName(tag) }); const pos = self.pos; self.advanceOne(); // TODO: find a better point to advance to self.diagnostics.append(self.gpa, .{ @@ -207,7 +208,7 @@ pub fn parseHeader(self: *Parser) IR.NodeStore.HeaderIdx { // .KwPackage => {}, // .KwHosted => {}, else => { - return self.store.addMalformed(IR.NodeStore.HeaderIdx, .missing_header, self.pos); + return self.pushMalformed(IR.NodeStore.HeaderIdx, .missing_header); }, } } @@ -217,32 +218,24 @@ fn parseModuleHeader(self: *Parser) IR.NodeStore.HeaderIdx { const start = self.pos; - self.advance(); + self.advance(); // Advance past KwModule // Get exposes - if (self.peek() != .OpenSquare) { + self.expect(.OpenSquare) catch { std.debug.panic("TODO: Handle header with no exposes open bracket: {s}", .{@tagName(self.peek())}); - } - self.advance(); - const scratch_top = self.store.scratchTokenTop(); - while (self.peek() != .CloseSquare) { - if (self.peek() != .LowerIdent and self.peek() != .UpperIdent) { - std.debug.panic("TODO: Handler header bad exposes contents: {s}", .{@tagName(self.peek())}); - } - - self.store.addScratchToken(self.pos); - self.advance(); - - if (self.peek() != .Comma) { - break; + }; + const scratch_top = self.store.scratchExposedItemTop(); + self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { + std.debug.print("Ended at {s}\n", .{@tagName(self.peek())}); + while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { + self.advance(); } - self.advance(); - } - if (self.peek() != .CloseSquare) { - std.debug.panic("TODO: Handle Bad header no closing exposes bracket: {s}", .{@tagName(self.peek())}); - } - const exposes = self.store.tokenSpanFrom(scratch_top); - self.advance(); + self.expect(.CloseSquare) catch {}; + self.store.clearScratchExposedItemsFrom(scratch_top); + std.debug.print("Expected CloseSquare, got {s}\n", .{@tagName(self.peek())}); + return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close); + }; + const exposes = self.store.exposedItemSpanFrom(scratch_top); return self.store.addHeader(.{ .module = .{ .region = .{ .start = start, .end = self.pos }, @@ -258,41 +251,30 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { var platform_name: ?TokenIdx = null; std.debug.assert(self.peek() == .KwApp); - self.advance(); + self.advance(); // Advance past KwApp // Get provides - if (self.peek() != .OpenSquare) { + self.expect(.OpenSquare) catch { return self.pushMalformed(IR.NodeStore.HeaderIdx, .expected_provides_open_square); - } - self.advance(); - const scratch_top = self.store.scratchTokenTop(); - while (self.peek() != .CloseSquare) { - if (self.peek() != .LowerIdent and self.peek() != .UpperIdent) { - self.store.clearScratchTokensFrom(scratch_top); - return self.pushMalformed(IR.NodeStore.HeaderIdx, .expected_provides); - } - - self.store.addScratchToken(self.pos); - self.advance(); - - if (self.peek() != .Comma) { - break; + }; + const scratch_top = self.store.scratchExposedItemTop(); + self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { + std.debug.print("Ended at {s}\n", .{@tagName(self.peek())}); + while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { + self.advance(); } - self.advance(); - } - if (self.peek() != .CloseSquare) { - self.store.clearScratchTokensFrom(scratch_top); - return self.pushMalformed(IR.NodeStore.HeaderIdx, .expected_provides_close_square); - } - const provides = self.store.tokenSpanFrom(scratch_top); - self.advance(); + self.expect(.CloseSquare) catch {}; + self.store.clearScratchExposedItemsFrom(scratch_top); + std.debug.print("Expected CloseSquare, got {s}\n", .{@tagName(self.peek())}); + return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close); + }; + const provides = self.store.exposedItemSpanFrom(scratch_top); // Get platform and packages const fields_scratch_top = self.store.scratchRecordFieldTop(); - if (self.peek() != .OpenCurly) { + self.expect(.OpenCurly) catch { return self.pushMalformed(IR.NodeStore.HeaderIdx, .expected_package_platform_open_curly); - } - self.advance(); + }; while (self.peek() != .CloseCurly) { const entry_start = self.pos; if (self.peek() != .LowerIdent) { @@ -362,6 +344,74 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { return self.pushMalformed(IR.NodeStore.HeaderIdx, .no_platform); } +pub fn parseIdent(self: *Parser) ?tokenize.Token.Idx { + if (self.peek() == .LowerIdent or self.peek() == .UpperIdent) { + const pos = self.pos; + self.advance(); + return pos; + } + return null; +} + +pub fn parseExposedItem(self: *Parser) IR.NodeStore.ExposedItemIdx { + const start = self.pos; + var end = start; + switch (self.peek()) { + .LowerIdent => { + var as: ?TokenIdx = null; + if (self.peekNext() == .KwAs) { + self.advance(); // Advance past LowerIdent + self.advance(); // Advance past KwAs + as = self.pos; + self.expect(.LowerIdent) catch { + return self.pushMalformed(IR.NodeStore.ExposedItemIdx, .unexpected_token); + }; + end = self.pos; + } else { + self.advance(); // Advance past LowerIdent + } + const ei = self.store.addExposedItem(.{ .lower_ident = .{ + .region = .{ .start = start, .end = end }, + .ident = start, + .as = as, + } }); + + return ei; + }, + .UpperIdent => { + var as: ?TokenIdx = null; + if (self.peekNext() == .KwAs) { + self.advance(); // Advance past UpperIdent + self.advance(); // Advance past KwAs + as = self.pos; + self.expect(.UpperIdent) catch { + return self.pushMalformed(IR.NodeStore.ExposedItemIdx, .unexpected_token); + }; + end = self.pos; + } else if (self.peekNext() == .DotStar) { + self.advance(); // Advance past UpperIdent + self.advance(); // Advance past DotStar + return self.store.addExposedItem(.{ .upper_ident_star = .{ + .region = .{ .start = start, .end = self.pos }, + .ident = start, + } }); + } else { + self.advance(); // Advance past UpperIdent + } + const ei = self.store.addExposedItem(.{ .upper_ident = .{ + .region = .{ .start = start, .end = end }, + .ident = start, + .as = as, + } }); + + return ei; + }, + else => { + return self.pushMalformed(IR.NodeStore.ExposedItemIdx, .unexpected_token); + }, + } +} + /// parse a roc statement /// /// e.g. `import Foo`, or `foo = 2 + x` @@ -369,21 +419,51 @@ pub fn parseStmt(self: *Parser) ?IR.NodeStore.StatementIdx { switch (self.peek()) { .KwImport => { const start = self.pos; - self.advance(); + self.advance(); // Advance past KwImport var qualifier: ?TokenIdx = null; + var alias_tok: ?TokenIdx = null; if (self.peek() == .LowerIdent) { qualifier = self.pos; - self.advance(); + self.advance(); // Advance past LowerIdent } if (self.peek() == .UpperIdent or (qualifier != null and self.peek() == .NoSpaceDotUpperIdent)) { + var exposes: IR.NodeStore.ExposedItemSpan = .{ .span = .{ .start = 0, .len = 0 } }; + const module_name_tok = self.pos; + if (self.peekNext() == .KwAs) { + self.advance(); // Advance past UpperIdent + self.advance(); // Advance past KwAs + alias_tok = self.pos; + self.expect(.UpperIdent) catch { + const malformed = self.pushMalformed(IR.NodeStore.StatementIdx, .unexpected_token); + self.advance(); + return malformed; + }; + } else if (self.peekNext() == .KwExposing) { + self.advance(); // Advance past ident + self.advance(); // Advance past KwExposing + self.expect(.OpenSquare) catch { + return self.pushMalformed(IR.NodeStore.StatementIdx, .import_exposing_no_open); + }; + const scratch_top = self.store.scratchExposedItemTop(); + self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { + while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { + self.advance(); + } + self.expect(.CloseSquare) catch {}; + self.store.clearScratchExposedItemsFrom(scratch_top); + return self.pushMalformed(IR.NodeStore.StatementIdx, .import_exposing_no_close); + }; + exposes = self.store.exposedItemSpanFrom(scratch_top); + } else { + self.advance(); // Advance past identifier + } const statement_idx = self.store.addStatement(.{ .import = .{ - .module_name_tok = self.pos, + .module_name_tok = module_name_tok, .qualifier_tok = qualifier, - .alias_tok = null, - .exposes = .{ .span = .{ .start = 0, .len = 0 } }, + .alias_tok = alias_tok, + .exposes = exposes, .region = .{ .start = start, .end = self.pos }, } }); - self.advance(); if (self.peek() == .Newline) { self.advance(); } @@ -683,7 +763,7 @@ pub fn parsePattern(self: *Parser, alternatives: Alternatives) IR.NodeStore.Patt } } if ((self.store.scratchPatternTop() - patterns_scratch_top) == 0) { - std.debug.panic("Should have gotten a valid pattern, pos={d} peek={s}", .{ self.pos, @tagName(self.peek()) }); + std.debug.panic("Should have gotten a valid pattern, pos={d} peek={s}\n", .{ self.pos, @tagName(self.peek()) }); } const patterns = self.store.patternSpanFrom(patterns_scratch_top); return self.store.addPattern(.{ .alternatives = .{ @@ -888,7 +968,7 @@ pub fn parseExprWithBp(self: *Parser, min_bp: u8) IR.NodeStore.ExprIdx { const condition = self.parseExpr(); const then = self.parseExpr(); if (self.peek() != .KwElse) { - std.debug.panic("TODO: problem for no else {s}@{d}", .{ @tagName(self.peek()), self.pos }); + return self.pushMalformed(IR.NodeStore.ExprIdx, .no_else); } self.advance(); const else_idx = self.parseExpr(); diff --git a/src/check/parse/tokenize.zig b/src/check/parse/tokenize.zig index badc5abf87..a0e1696cca 100644 --- a/src/check/parse/tokenize.zig +++ b/src/check/parse/tokenize.zig @@ -104,6 +104,7 @@ pub const Token = struct { Dot, DoubleDot, TripleDot, + DotStar, OpColon, OpArrow, OpFatArrow, @@ -118,6 +119,7 @@ pub const Token = struct { KwElse, KwExpect, KwExposes, + KwExposing, KwGenerates, KwHas, KwHosted, @@ -150,6 +152,7 @@ pub const Token = struct { .{ "else", .KwElse }, .{ "expect", .KwExpect }, .{ "exposes", .KwExposes }, + .{ "exposing", .KwExposing }, .{ "generates", .KwGenerates }, .{ "has", .KwHas }, .{ "hosted", .KwHosted }, @@ -196,6 +199,7 @@ pub const Token = struct { .KwElse, .KwExpect, .KwExposes, + .KwExposing, .KwGenerates, .KwHas, .KwHosted, @@ -898,6 +902,9 @@ pub const Tokenizer = struct { } else if (n == open_curly) { self.cursor.pos += 1; self.output.pushTokenNormal(.Dot, start, 1); + } else if (n == '*') { + self.cursor.pos += 2; + self.output.pushTokenNormal(.DotStar, start, 2); } else { self.cursor.pos += 1; self.output.pushTokenNormal(.Dot, start, 1); @@ -1720,6 +1727,11 @@ fn rebuildBufferForTesting(buf: []const u8, tokens: *TokenizedBuffer, alloc: std try buf2.append(alloc, '.'); try buf2.append(alloc, '.'); }, + .DotStar => { + std.debug.assert(length == 2); + try buf2.append(alloc, '.'); + try buf2.append(alloc, '*'); + }, .OpColon => { std.debug.assert(length == 1); try buf2.append(alloc, ':'); @@ -1763,6 +1775,9 @@ fn rebuildBufferForTesting(buf: []const u8, tokens: *TokenizedBuffer, alloc: std .KwExposes => { try buf2.appendSlice(alloc, "exposes"); }, + .KwExposing => { + try buf2.appendSlice(alloc, "exposing"); + }, .KwGenerates => { try buf2.appendSlice(alloc, "generates"); }, diff --git a/src/fmt.zig b/src/fmt.zig index 92f1eb7e1b..6e45a2d9f6 100644 --- a/src/fmt.zig +++ b/src/fmt.zig @@ -77,6 +77,22 @@ fn formatStatement(fmt: *Formatter, si: StatementIdx) NewlineBehavior { .import => |i| { fmt.buffer.appendSlice(fmt.gpa, "import ") catch |err| exitOnOom(err); fmt.formatIdent(i.module_name_tok, i.qualifier_tok); + if (i.alias_tok) |a| { + fmt.pushAll(" as "); + fmt.pushTokenText(a); + } + if (i.exposes.span.len > 0) { + fmt.pushAll(" exposing ["); + var n: usize = 0; + for (fmt.ast.store.exposedItemSlice(i.exposes)) |tok| { + fmt.formatExposedItem(tok); + if (n < i.exposes.span.len - 1) { + fmt.pushAll(", "); + } + n += 1; + } + fmt.push(']'); + } return .extra_newline_needed; }, .type_decl => |d| { @@ -385,6 +401,30 @@ fn formatPattern(fmt: *Formatter, pi: PatternIdx) void { } } +fn formatExposedItem(fmt: *Formatter, idx: IR.NodeStore.ExposedItemIdx) void { + const item = fmt.ast.store.getExposedItem(idx); + switch (item) { + .lower_ident => |i| { + fmt.pushTokenText(i.ident); + if (i.as) |a| { + fmt.pushAll(" as "); + fmt.pushTokenText(a); + } + }, + .upper_ident => |i| { + fmt.pushTokenText(i.ident); + if (i.as) |a| { + fmt.pushAll(" as "); + fmt.pushTokenText(a); + } + }, + .upper_ident_star => |i| { + fmt.pushTokenText(i.ident); + fmt.pushAll(".*"); + }, + } +} + fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { const header = fmt.ast.store.getHeader(hi); switch (header) { @@ -392,10 +432,9 @@ fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { fmt.pushAll("app ["); // Format provides var i: usize = 0; - for (fmt.ast.store.tokenSlice(a.provides)) |p| { - fmt.pushTokenText(p); - i += 1; - if (i < a.provides.span.len) { + for (fmt.ast.store.exposedItemSlice(a.provides)) |p| { + fmt.formatExposedItem(p); + if (i < a.provides.span.len - 1) { fmt.pushAll(", "); } } @@ -425,8 +464,8 @@ fn formatHeader(fmt: *Formatter, hi: HeaderIdx) void { .module => |m| { fmt.pushAll("module ["); var i: usize = 0; - for (fmt.ast.store.tokenSlice(m.exposes)) |p| { - fmt.pushTokenText(p); + for (fmt.ast.store.exposedItemSlice(m.exposes)) |p| { + fmt.formatExposedItem(p); i += 1; if (i < m.exposes.span.len) { fmt.pushAll(", "); @@ -836,7 +875,11 @@ test "Syntax grab bag" { try moduleFmtsSame( \\app [main!] { pf: platform "../basic-cli/platform.roc" } \\ - \\import pf.Stdout + \\import pf.Stdout exposing [line!, write!] + \\ + \\import Something exposing [func as function, Type as ValueCategory, Custom.*] + \\ + \\import BadName as GoodName \\ \\Map a b : List(a), (a -> b) -> List(b) \\ From 3f0e8af309f6558639e8e82a95c03ae10051b834 Mon Sep 17 00:00:00 2001 From: Anthony Bullard Date: Mon, 10 Mar 2025 08:52:13 -0500 Subject: [PATCH 2/5] Update parser progress summary --- src/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/README.md b/src/README.md index f27746a9f5..d5285c9ad8 100644 --- a/src/README.md +++ b/src/README.md @@ -4,7 +4,7 @@ This table provides a summary of progress for the zig compiler re-write and shou | | Str & Num * | Functions | Modules | Collections | Records & Tuples | Recursive Types | Static Dispatch | |--------------------------|:-----------:|:----------:|:-------:|:-----------:|:-----------------:|:----------------:|:----------------:| -| **Parse** | 🚧 | 🚧 | | | | | | +| **Parse** | 🪫 | 🪫 | 🚧 | 🪫 | 🪫 | 🪫 | 🪫 | | **Canonicalize** | 🚧 | 🚧 | | | | | | | **Resolve Imports** | | | | | | | | | **Check Types** | | | | | | | | From 9383dc83dc4cbe48e8ef0244c8c4e7a4dbccd4a1 Mon Sep 17 00:00:00 2001 From: Anthony Bullard Date: Mon, 10 Mar 2025 17:10:34 -0500 Subject: [PATCH 3/5] Fix lint error --- src/check/parse/Parser.zig | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/check/parse/Parser.zig b/src/check/parse/Parser.zig index de24d77308..6efc8ac102 100644 --- a/src/check/parse/Parser.zig +++ b/src/check/parse/Parser.zig @@ -344,15 +344,7 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { return self.pushMalformed(IR.NodeStore.HeaderIdx, .no_platform); } -pub fn parseIdent(self: *Parser) ?tokenize.Token.Idx { - if (self.peek() == .LowerIdent or self.peek() == .UpperIdent) { - const pos = self.pos; - self.advance(); - return pos; - } - return null; -} - +/// Parses an ExposedItem, adding it to the NodeStore and returning the Idx pub fn parseExposedItem(self: *Parser) IR.NodeStore.ExposedItemIdx { const start = self.pos; var end = start; From f2fc44b793df7d74cd2618cf79e21eeefd2d1e9a Mon Sep 17 00:00:00 2001 From: Anthony Bullard Date: Mon, 10 Mar 2025 17:32:38 -0500 Subject: [PATCH 4/5] New snapshots --- src/snapshots/001.txt | 3 ++- src/snapshots/003.txt | 3 ++- src/snapshots/add_var_with_spaces.txt | 3 ++- src/snapshots/if_then_else.txt | 3 ++- src/snapshots/some_folder/002.txt | 4 +++- src/snapshots/type_declarations.txt | 14 +++++++------- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/snapshots/001.txt b/src/snapshots/001.txt index f9329fff63..3785221d64 100644 --- a/src/snapshots/001.txt +++ b/src/snapshots/001.txt @@ -15,7 +15,8 @@ module [foo] foo = "one" ~~~PARSE (file - (header 'foo') + (header + (exposed_item (lower_ident 'foo'))) (decl (ident 'foo') (string 'one'))) \ No newline at end of file diff --git a/src/snapshots/003.txt b/src/snapshots/003.txt index 89f25379db..f9faee8b94 100644 --- a/src/snapshots/003.txt +++ b/src/snapshots/003.txt @@ -14,7 +14,8 @@ expect 1 == 1 return 2 ~~~PARSE (file - (header 'decoder') + (header + (exposed_item (lower_ident 'decoder'))) (import 'json' '.Json' diff --git a/src/snapshots/add_var_with_spaces.txt b/src/snapshots/add_var_with_spaces.txt index e575ec14c7..96e6d4d023 100644 --- a/src/snapshots/add_var_with_spaces.txt +++ b/src/snapshots/add_var_with_spaces.txt @@ -10,7 +10,8 @@ module [add2] add2 = x + 2 ~~~PARSE (file - (header 'add2') + (header + (exposed_item (lower_ident 'add2'))) (decl (ident 'add2') (binop diff --git a/src/snapshots/if_then_else.txt b/src/snapshots/if_then_else.txt index 22ff0634e2..1b22c272cd 100644 --- a/src/snapshots/if_then_else.txt +++ b/src/snapshots/if_then_else.txt @@ -14,7 +14,8 @@ module [foo] foo = if true A else B ~~~PARSE (file - (header 'foo') + (header + (exposed_item (lower_ident 'foo'))) (decl (ident 'foo') (if_then_else diff --git a/src/snapshots/some_folder/002.txt b/src/snapshots/some_folder/002.txt index 9cdffe412d..777310f844 100644 --- a/src/snapshots/some_folder/002.txt +++ b/src/snapshots/some_folder/002.txt @@ -8,7 +8,9 @@ foo = "one" bar = "two" ~~~PARSE (file - (header 'foo' 'bar') + (header + (exposed_item (lower_ident 'foo')) + (exposed_item (lower_ident 'bar'))) (decl (ident 'foo') (string 'one')) diff --git a/src/snapshots/type_declarations.txt b/src/snapshots/type_declarations.txt index c6f9bb589d..e44bfd40d7 100644 --- a/src/snapshots/type_declarations.txt +++ b/src/snapshots/type_declarations.txt @@ -15,13 +15,13 @@ SomeFunc a : Maybe(a), a -> Maybe(a) ~~~PARSE (file (header - 'Map' - 'Foo' - 'Some' - 'Maybe' - 'SomeFunc' - 'add_one' - 'main!') + (exposed_item (upper_ident 'Map')) + (exposed_item (upper_ident 'Foo')) + (exposed_item (upper_ident 'Some')) + (exposed_item (upper_ident 'Maybe')) + (exposed_item (upper_ident 'SomeFunc')) + (exposed_item (lower_ident 'add_one')) + (exposed_item (lower_ident 'main!'))) (type_decl (header 'Map' From 45c51f9fdba4dfdf8aed4d8fc0f74fc88385f061 Mon Sep 17 00:00:00 2001 From: Anthony Bullard Date: Mon, 10 Mar 2025 17:39:35 -0500 Subject: [PATCH 5/5] Remove panic --- src/check/parse/Parser.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/check/parse/Parser.zig b/src/check/parse/Parser.zig index 6efc8ac102..0b9ae3c471 100644 --- a/src/check/parse/Parser.zig +++ b/src/check/parse/Parser.zig @@ -116,7 +116,6 @@ pub fn pushDiagnostic(self: *Parser, tag: IR.Diagnostic.Tag, region: IR.Region) } /// add a malformed token pub fn pushMalformed(self: *Parser, comptime t: type, tag: IR.Diagnostic.Tag) t { - std.debug.panic("!!!! pushMalformed t={any} tag={s}\n", .{ t, @tagName(tag) }); const pos = self.pos; self.advanceOne(); // TODO: find a better point to advance to self.diagnostics.append(self.gpa, .{ @@ -226,13 +225,11 @@ fn parseModuleHeader(self: *Parser) IR.NodeStore.HeaderIdx { }; const scratch_top = self.store.scratchExposedItemTop(); self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { - std.debug.print("Ended at {s}\n", .{@tagName(self.peek())}); while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { self.advance(); } self.expect(.CloseSquare) catch {}; self.store.clearScratchExposedItemsFrom(scratch_top); - std.debug.print("Expected CloseSquare, got {s}\n", .{@tagName(self.peek())}); return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close); }; const exposes = self.store.exposedItemSpanFrom(scratch_top); @@ -259,13 +256,11 @@ pub fn parseAppHeader(self: *Parser) IR.NodeStore.HeaderIdx { }; const scratch_top = self.store.scratchExposedItemTop(); self.parseCollectionSpan(IR.NodeStore.ExposedItemIdx, .CloseSquare, IR.NodeStore.addScratchExposedItem, Parser.parseExposedItem) catch { - std.debug.print("Ended at {s}\n", .{@tagName(self.peek())}); while (self.peek() != .CloseSquare and self.peek() != .EndOfFile) { self.advance(); } self.expect(.CloseSquare) catch {}; self.store.clearScratchExposedItemsFrom(scratch_top); - std.debug.print("Expected CloseSquare, got {s}\n", .{@tagName(self.peek())}); return self.pushMalformed(IR.NodeStore.HeaderIdx, .import_exposing_no_close); }; const provides = self.store.exposedItemSpanFrom(scratch_top);