Merge pull request #7672 from roc-lang/fuzz-fixes

Parsing Fuzzer Fixes

Author: lukewilliamboswell

crates/compiler/builtins/bitcode/build.zig

@@ -119,9 +119,9 @@ fn generateObjectFile(
 
     const suffix =
         if (target.result.os.tag == std.Target.Os.Tag.windows)
-        "obj"
-    else
-        "o";
+            "obj"
+        else
+            "o";
     const install = b.addInstallFile(obj_file, b.fmt("{s}.{s}", .{ object_name, suffix }));
 
     const obj_step = b.step(step_name, "Build object file for linking");

src/base/sexpr.zig

@@ -129,7 +129,7 @@ pub const Expr = union(enum) {
 
                 try writer.print(")", .{});
             },
-            .string => |s| try writer.print("'{s}'", .{s}),
+            .string => |s| try writer.print("\"{s}\"", .{s}),
             .signed_int => |i| try writer.print("{d}", .{i}),
             .unsigned_int => |u| try writer.print("{d}", .{u}),
             .float => |f| try writer.print("{any}", .{f}),
@@ -191,7 +191,7 @@ test "s-expression" {
         foo.toStringPretty(buf.writer().any());
         const expected =
             \\(foo
-            \\    'bar'
+            \\    "bar"
             \\    -123
             \\    (baz 456 7.89e2))
         ;

src/check/parse.zig

@@ -22,15 +22,19 @@ pub fn parse(env: *base.ModuleEnv, source: []const u8) IR {
     tokenizer.tokenize();
     const result = tokenizer.finishAndDeinit();
 
-    if (result.messages.len > 0) {
-        tokenizeReport(env.gpa, source, result.messages);
+    for (result.messages) |msg| {
+        _ = env.problems.append(env.gpa, .{ .tokenize = msg });
     }
 
     var parser = Parser.init(result.tokens);
     defer parser.deinit();
 
     parser.parseFile();
 
+    for (parser.diagnostics.items) |msg| {
+        _ = env.problems.append(env.gpa, .{ .parser = msg });
+    }
+
     const errors = parser.diagnostics.toOwnedSlice(env.gpa) catch |err| exitOnOom(err);
 
     return .{
@@ -40,106 +44,3 @@ pub fn parse(env: *base.ModuleEnv, source: []const u8) IR {
         .errors = errors,
     };
 }
-
-fn lineNum(newlines: std.ArrayList(usize), pos: u32) u32 {
-    const pos_usize = @as(usize, @intCast(pos));
-    var lineno: u32 = 0;
-    while (lineno < newlines.items.len) {
-        if (newlines.items[lineno + 1] > pos_usize) {
-            return lineno;
-        }
-        lineno += 1;
-    }
-    return lineno;
-}
-
-fn tokenizeReport(allocator: std.mem.Allocator, source: []const u8, msgs: []const tokenize.Diagnostic) void {
-    std.debug.print("Found the {d} following issues while tokenizing:\n", .{msgs.len});
-    var newlines = std.ArrayList(usize).init(allocator);
-    defer newlines.deinit();
-    newlines.append(0) catch |err| exitOnOom(err);
-    var pos: usize = 0;
-    for (source) |c| {
-        if (c == '\n') {
-            newlines.append(pos) catch |err| exitOnOom(err);
-        }
-        pos += 1;
-    }
-    for (msgs) |message| {
-        switch (message.tag) {
-            .MismatchedBrace => {
-                const start_line_num = lineNum(newlines, message.begin);
-                const start_col = message.begin - newlines.items[start_line_num];
-                const end_line_num = lineNum(newlines, message.end);
-                const end_col = message.end - newlines.items[end_line_num];
-
-                const src = source[newlines.items[start_line_num]..newlines.items[end_line_num + 1]];
-                var spaces = std.ArrayList(u8).init(allocator);
-                defer spaces.deinit();
-                for (0..start_col) |_| {
-                    spaces.append(' ') catch |err| exitOnOom(err);
-                }
-
-                std.debug.print(
-                    "({d}:{d}-{d}:{d}) Expected the correct closing brace here:\n{s}\n{s}^\n",
-                    .{ start_line_num, start_col, end_line_num, end_col, src, spaces.toOwnedSlice() catch |err| exitOnOom(err) },
-                );
-            },
-            else => {
-                std.debug.print("MSG: {any}\n", .{message});
-            },
-        }
-    }
-}
-
-// TODO move this somewhere better, for now it's here to keep it simple.
-fn testSExprHelper(source: []const u8, expected: []const u8) !void {
-    var env = base.ModuleEnv.init(testing.allocator);
-    defer env.deinit();
-
-    // parse our source
-    var parse_ast = parse(&env, source);
-    defer parse_ast.deinit();
-    std.testing.expectEqualSlices(IR.Diagnostic, &[_]IR.Diagnostic{}, parse_ast.errors) catch {
-        std.debug.print("Tokens:\n{any}", .{parse_ast.tokens.tokens.items(.tag)});
-        std.debug.panic("Test failed with parse errors", .{});
-    };
-
-    // shouldn't be required in future
-    parse_ast.store.emptyScratch();
-
-    // buffer to write our SExpr to
-    var buf = std.ArrayList(u8).init(testing.allocator);
-    defer buf.deinit();
-
-    // convert the AST to our SExpr
-    try parse_ast.toSExprStr(&env, buf.writer().any());
-
-    // TODO in future we should just write the SExpr to a file and snapshot it
-    // for now we are comparing strings to keep it simple
-    try testing.expectEqualStrings(expected, buf.items[0..]);
-}
-
-test "example s-expr" {
-    const source =
-        \\module [foo, bar]
-        \\
-        \\foo = "hey"
-        \\bar = "yo"
-    ;
-
-    const expected =
-        \\(file
-        \\    (header
-        \\        (exposed_item (lower_ident 'foo'))
-        \\        (exposed_item (lower_ident 'bar')))
-        \\    (decl
-        \\        (ident 'foo')
-        \\        (string 'hey'))
-        \\    (decl
-        \\        (ident 'bar')
-        \\        (string 'yo')))
-    ;
-
-    try testSExprHelper(source, expected);
-}