From 94c97edbd3e627373dda5bca510e646c9931d495 Mon Sep 17 00:00:00 2001 From: Jefferson Oliveira Date: Thu, 21 May 2026 09:08:17 -0300 Subject: [PATCH 01/11] refactor(parser): update parser generation to Go and add Go parser implementation - Changed the parser generation step in `build.zig` to produce a Go parser instead of a C parser. - Introduced a new `parser.go` file containing the generated Go parser code, implementing the necessary structures and methods for parsing. - Updated output paths and command options to align with Go conventions, including package naming and file structure. --- build.zig | 12 +- lib/parser/parser.go | 2166 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2173 insertions(+), 5 deletions(-) create mode 100644 lib/parser/parser.go diff --git a/build.zig b/build.zig index 0ea11b5..d718733 100644 --- a/build.zig +++ b/build.zig @@ -7,7 +7,7 @@ pub fn build(b: *std.Build) void { const optimize = b.standardOptimizeOption(.{}); - const parser_gen = b.step("parser:gen", "Generate C Parser"); + const parser_gen = b.step("parser:gen", "Generate Go Parser"); const generate_parser_cmd = b.addSystemCommand(&.{ "langlang", @@ -15,11 +15,13 @@ pub fn build(b: *std.Build) void { GRAMMAR_FILE, "-disable-capture-spaces", "-output-language", - "c", + "go", "-output-path", - "lib/parser.c", - "-c-header-path", - "lib/parser.h", + "lib/parser/parser.go", + "-go-package", + "parser", + "-go-parser", + "Parser", }); parser_gen.dependOn(&generate_parser_cmd.step); diff --git a/lib/parser/parser.go b/lib/parser/parser.go new file mode 100644 index 0000000..f303db2 --- /dev/null +++ b/lib/parser/parser.go @@ -0,0 +1,2166 @@ +// Code generated by langlang (unknown commit hash), DO NOT EDIT. +// Source File: lib/grammar/proto-circ.peg +package parser + +import ( + "encoding/hex" + "fmt" + "math/bits" + "sort" + "strconv" + "strings" + "unicode/utf8" +) + +type Matcher interface { + Match([]byte) (Tree, int, error) + + SourceMap() *SourceMap +} +type NodeID uint32 +type NodeType uint8 + +const ( + NodeType_String NodeType = iota + + NodeType_Sequence + + NodeType_Node + + NodeType_Error +) + +type Location struct { + Line int + Column int + Cursor int +} +type Span struct { + Start Location + End Location +} +type Tree interface { + Root() (NodeID, bool) + + Visit(id NodeID, fn func(NodeID) bool) + + Type(NodeID) NodeType + + Span(NodeID) Span + + Location(cursor int) Location + + CursorU16(cursor int) int + + Name(NodeID) string + + Message(NodeID) string + + Child(NodeID) (NodeID, bool) + + Children(NodeID) []NodeID + + Text(NodeID) string + + Pretty(NodeID) string + + Highlight(NodeID) string + + Copy() Tree +} +type ImportLoader interface { + GetPath(importPath, parentPath string) (string, error) + + GetContent(path string) ([]byte, error) +} +type FileID int +type SourceLocation struct { + FileID FileID + Span Span +} +type DiagnosticSeverity int + +const ( + DiagnosticError DiagnosticSeverity = iota + DiagnosticWarning + DiagnosticInfo + DiagnosticHint +) + +type Diagnostic struct { + Location SourceLocation + Severity DiagnosticSeverity + Message string + Code string + FilePath string + Expected []ErrHint +} +type SourceMap struct { + Data []byte + Files []string + entries []srcMapEntry +} + +var bytecodeForParser = &Bytecode{ + code: []byte{ + 11, 5, 0, 0, 0, 15, 1, 0, 6, 231, 0, 18, 0, 0, 6, 33, 0, 26, 3, 0, 2, 47, 0, 2, 47, 0, 18, 1, 0, 27, 8, 228, 0, 6, 103, 0, 15, 4, 0, 18, 0, 0, 2, 105, 0, 2, 109, 0, 2, 112, 0, 2, 111, 0, 2, 114, 0, 2, 116, 0, 19, 6, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 18, 0, 0, 17, 4, 0, 19, 1, 0, 25, 18, 5, 0, 27, 18, 0, 0, 17, 4, 0, 19, 1, 0, 16, 8, 228, 0, 6, 198, 0, 15, 7, 0, 6, 139, 0, 11, 252, 0, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 18, 0, 0, 11, 10, 2, 0, 8, 194, 0, 11, 252, 0, 0, 18, 0, 0, 15, 10, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 6, 193, 0, 18, 0, 0, 18, 0, 0, 17, 6, 0, 19, 1, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 9, 165, 0, 16, 16, 8, 228, 0, 15, 11, 0, 18, 0, 0, 11, 188, 1, 0, 18, 0, 0, 2, 60, 0, 2, 62, 0, 19, 2, 0, 18, 0, 0, 11, 188, 1, 0, 16, 9, 11, 0, 16, 12, 15, 13, 0, 18, 0, 0, 11, 252, 0, 0, 18, 0, 0, 11, 10, 2, 0, 16, 12, 15, 8, 0, 6, 23, 1, 2, 105, 0, 2, 110, 0, 2, 112, 0, 2, 117, 0, 2, 116, 0, 19, 5, 0, 8, 186, 1, 6, 50, 1, 2, 111, 0, 2, 117, 0, 2, 116, 0, 2, 112, 0, 2, 117, 0, 2, 116, 0, 19, 6, 0, 8, 186, 1, 6, 68, 1, 2, 98, 0, 2, 117, 0, 2, 115, 0, 19, 3, 0, 8, 186, 1, 6, 86, 1, 2, 108, 0, 2, 101, 0, 2, 100, 0, 19, 3, 0, 8, 186, 1, 6, 104, 1, 2, 97, 0, 2, 110, 0, 2, 100, 0, 19, 3, 0, 8, 186, 1, 6, 119, 1, 2, 111, 0, 2, 114, 0, 19, 2, 0, 8, 186, 1, 6, 137, 1, 2, 110, 0, 2, 111, 0, 2, 116, 0, 19, 3, 0, 8, 186, 1, 6, 155, 1, 2, 120, 0, 2, 111, 0, 2, 114, 0, 19, 3, 0, 8, 186, 1, 6, 176, 1, 2, 110, 0, 2, 97, 0, 2, 110, 0, 2, 100, 0, 19, 4, 0, 8, 186, 1, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 16, 12, 15, 12, 0, 6, 223, 1, 11, 233, 0, 0, 18, 0, 0, 17, 7, 0, 19, 1, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 8, 8, 2, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 18, 0, 0, 6, 8, 2, 18, 0, 0, 17, 7, 0, 19, 1, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 8, 8, 2, 16, 12, 15, 9, 0, 18, 0, 0, 17, 8, 0, 19, 1, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 18, 0, 0, 17, 9, 0, 19, 1, 0, 18, 0, 0, 11, 188, 1, 0, 6, 98, 2, 18, 0, 0, 18, 0, 0, 17, 6, 0, 19, 1, 0, 18, 0, 0, 26, 6, 0, 17, 2, 0, 18, 3, 0, 27, 18, 0, 0, 17, 9, 0, 19, 1, 0, 18, 0, 0, 11, 188, 1, 0, 9, 54, 2, 18, 0, 0, 17, 10, 0, 19, 1, 0, 16, 12, + }, + strs: []string{ + "", "Program", "Spacing", "Annotation", "ImportDecl", "", "Identifier", "Declaration", "ComponentType", "BusType", "IdentList", "Connection", "PortRef", "AnonDecl", "Space", "EOL", "EOF", + }, + rxps: map[int]int{}, + rxbs: bitset512{ + 0, 0, 0, 0, 0, 0, 0, 0, + }, + + smap: map[string]int{ + "": 5, "Annotation": 3, "AnonDecl": 13, "BusType": 9, "ComponentType": 8, "Connection": 11, "Declaration": 7, "EOF": 16, "EOL": 15, "IdentList": 10, "Identifier": 6, "ImportDecl": 4, "PortRef": 12, "Program": 1, "Space": 14, "Spacing": 2, + }, + sets: []charset{ + {bits: [32]byte{0, 38, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{255, 251, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}, + {bits: [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 254, 255, 255, 135, 254, 255, 255, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 0, 0, 255, 3, 254, 255, 255, 135, 254, 255, 255, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{255, 255, 255, 255, 251, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}, + {bits: [32]byte{0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {bits: [32]byte{0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + }, + sexp: [][]expected{ + {{a: '\t'}, {a: '\n'}, {a: '\r'}, {a: ' '}}, + {}, + {{a: 'A', b: 'Z'}, {a: '_'}, {a: 'a', b: 'z'}}, + {{a: '0', b: '9'}, {a: 'A', b: 'Z'}, {a: '_'}, {a: 'a', b: 'z'}}, + {{a: '"'}}, + {}, + {{a: ','}}, + {{a: '.'}}, + {{a: '('}}, + {{a: '='}}, + {{a: ')'}}, + }, +} + +type Parser struct { + input []byte + vm *virtualMachine +} + +func NewParser() *Parser { + vm := NewVirtualMachine(bytecodeForParser) + return &Parser{vm: vm} +} +func (p *Parser) ParseProgram() (Tree, error) { return p.parseFn(5) } +func (p *Parser) ParseAnonDecl() (Tree, error) { return p.parseFn(233) } +func (p *Parser) ParseComponentType() (Tree, error) { return p.parseFn(252) } +func (p *Parser) ParsePortRef() (Tree, error) { return p.parseFn(444) } +func (p *Parser) ParseBusType() (Tree, error) { return p.parseFn(522) } +func (p *Parser) Parse() (Tree, error) { return p.parseFn(5) } +func (p *Parser) SetInput(input []byte) { p.input = input } +func (p *Parser) GetInput() []byte { return p.input } +func (p *Parser) SetLabelMessages(el map[int]int) { p.vm.SetLabelMessages(el) } +func (p *Parser) SetShowFails(v bool) { p.vm.SetShowFails(v) } + +func LabelMessagesForParser(labels map[string]string) map[int]int { + return bytecodeForParser.CompileErrorLabels(labels) +} +func (p *Parser) parseFn(addr int) (Tree, error) { + val, _, err := p.vm.MatchRule(p.input, addr) + return val, err +} + +const eof = -1 + +func NewLocation(line, column, cursor int) Location { + return Location{Line: line, Column: column, Cursor: cursor} +} +func (l Location) String() string { + return fmt.Sprintf("%d:%d", l.Line, l.Column) +} +func NewSpan(start, end Location) Span { + return Span{Start: start, End: end} +} +func (s Span) Contains(other Span) bool { + return s.Start.Cursor <= other.Start.Cursor && s.End.Cursor >= other.End.Cursor +} +func (s Span) String() string { + startLoc := s.Start + endLoc := s.End + startLine, startCol := int(startLoc.Line), int(startLoc.Column) + endLine, endCol := int(endLoc.Line), int(endLoc.Column) + if startLine == endLine && endLine == startCol && endCol == endLine && startLine == 0 { + startLine++ + endLine++ + startCol++ + endCol++ + } + if startLine == endLine && startLine == 1 { + if startCol == endCol { + return fmt.Sprintf("%d", startCol) + } + return fmt.Sprintf("%d..%d", startCol, endCol) + } + if startLine == endLine && startCol == endCol { + return fmt.Sprintf("%d:%d", startLine, startCol) + } + return fmt.Sprintf("%d:%d..%d:%d", startLine, startCol, endLine, endCol) +} + +type srcMapEntry struct { + offset int + fileID int + startLine, startCol, startCursor int + endLine, endCol, endCursor int +} +type posIndex struct { + input []byte + + lineStart []int + + runeUnits, u16Units *unitsIndex +} + +func newPosIndex(input []byte) *posIndex { + + lineStart := make([]int, 1, 64) + lineStart[0] = 0 + for i, b := range input { + if b == '\n' { + + lineStart = append(lineStart, i+1) + } + } + return &posIndex{input: input, lineStart: lineStart} +} +func (pi *posIndex) LocationAt(cursor int) Location { + if cursor < 0 { + cursor = 0 + } + if cursor > len(pi.input) { + cursor = len(pi.input) + } + + lineIdx := sort.Search(len(pi.lineStart), func(i int) bool { + return pi.lineStart[i] > cursor + }) - 1 + if lineIdx < 0 { + lineIdx = 0 + } + + lineStart := pi.lineStart[lineIdx] + + pi.ensureRuneUnits() + + col := pi.runeUnits.UnitsAt(cursor) - pi.runeUnits.UnitsAt(lineStart) + 1 + + return Location{ + Line: lineIdx + 1, + Column: col, + Cursor: cursor, + } +} +func (pi *posIndex) CursorAt(line0, col0 int) int { + + line := line0 + 1 + col := col0 + 1 + + if line < 1 { + line = 1 + } + if col < 1 { + col = 1 + } + + lineIdx := line - 1 + if lineIdx >= len(pi.lineStart) { + + return len(pi.input) + } + + lineStart := pi.lineStart[lineIdx] + + var lineEnd int + if lineIdx+1 < len(pi.lineStart) { + lineEnd = pi.lineStart[lineIdx+1] + } else { + lineEnd = len(pi.input) + } + + pi.ensureRuneUnits() + + startRunes := pi.runeUnits.UnitsAt(lineStart) + targetRunes := startRunes + col - 1 + + cursor := sort.Search(lineEnd-lineStart, func(i int) bool { + return pi.runeUnits.UnitsAt(lineStart+i) >= targetRunes + }) + lineStart + + if cursor > lineEnd { + cursor = lineEnd + } + + return cursor +} +func (pi *posIndex) CursorU16(cursor int) int { + pi.ensureU16Units() + return pi.u16Units.UnitsAt(cursor) +} +func (pi *posIndex) ensureRuneUnits() { + if pi.runeUnits == nil { + pi.runeUnits = newUnitsIndex(pi.input, unitsModeRune) + } +} +func (pi *posIndex) ensureU16Units() { + if pi.u16Units == nil { + pi.u16Units = newUnitsIndex(pi.input, unitsModeUTF16) + } +} + +type unitsMode uint8 + +const ( + unitsModeRune unitsMode = iota + unitsModeUTF16 +) + +type unitsIndex struct { + input []byte + + byteOffsets []int + + unitOffsets []int + + mode unitsMode +} + +func newUnitsIndex(input []byte, mode unitsMode) *unitsIndex { + + const strideBytes = 64 + + var ( + unitCount = 0 + bytesSinceCheckpoint = 0 + index = &unitsIndex{ + input: input, + byteOffsets: make([]int, 0, 128), + unitOffsets: make([]int, 0, 128), + mode: mode, + } + ) + + index.byteOffsets = append(index.byteOffsets, 0) + index.unitOffsets = append(index.unitOffsets, 0) + + for i := 0; i < len(input); { + r, size := utf8.DecodeRune(input[i:]) + if size <= 0 { + size = 1 + r = utf8.RuneError + } + + i += size + unitCount += index.unitsForRune(r) + bytesSinceCheckpoint += size + + if bytesSinceCheckpoint >= strideBytes { + index.byteOffsets = append(index.byteOffsets, i) + index.unitOffsets = append(index.unitOffsets, unitCount) + bytesSinceCheckpoint = 0 + } + } + + if last := index.byteOffsets[len(index.byteOffsets)-1]; last != len(input) { + index.byteOffsets = append(index.byteOffsets, len(input)) + index.unitOffsets = append(index.unitOffsets, unitCount) + } + return index +} +func (ix *unitsIndex) UnitsAt(cursor int) int { + if cursor < 0 { + cursor = 0 + } + if cursor > len(ix.input) { + cursor = len(ix.input) + } + + i := sort.Search(len(ix.byteOffsets), func(i int) bool { + return ix.byteOffsets[i] > cursor + }) - 1 + if i < 0 { + i = 0 + } + + bytePos := ix.byteOffsets[i] + unitPos := ix.unitOffsets[i] + + for bytePos < cursor { + r, size := utf8.DecodeRune(ix.input[bytePos:]) + if size <= 0 { + size = 1 + r = utf8.RuneError + } + if bytePos+size > cursor { + break + } + unitPos += ix.unitsForRune(r) + bytePos += size + } + return unitPos +} +func (ix *unitsIndex) unitsForRune(r rune) int { + switch ix.mode { + case unitsModeRune: + return 1 + case unitsModeUTF16: + if r > 0xFFFF { + return 2 + } + return 1 + default: + return 1 + } +} + +type FormatToken int + +const ( + FormatToken_None FormatToken = iota + FormatToken_Range + FormatToken_Literal + FormatToken_Error +) + +func (nt NodeType) String() string { + switch nt { + case NodeType_String: + return "string" + case NodeType_Sequence: + return "sequence" + case NodeType_Node: + return "node" + case NodeType_Error: + return "error" + default: + return "unknown" + } +} + +type node struct { + typ NodeType + start int + end int + nameID int32 + childID int32 + messageID int32 +} +type tree struct { + nodes []node + children []NodeID + childRanges []struct{ start, end int32 } + strs []string + input []byte + root NodeID + posView *posIndex +} + +func (t *tree) bindInput(input []byte) { t.input = input } +func (t *tree) bindStrings(strs []string) { t.strs = strs } +func (t *tree) reset() { + t.nodes = t.nodes[:0] + t.children = t.children[:0] + t.childRanges = t.childRanges[:0] + t.posView = nil + t.root = 0 +} +func (t *tree) Root() (NodeID, bool) { return t.root, int(t.root) < len(t.nodes) } +func (t *tree) SetRoot(id NodeID) { t.root = id } +func (t *tree) Type(id NodeID) NodeType { return t.nodes[id].typ } +func (t *tree) MessageID(id NodeID) int32 { return t.nodes[id].messageID } +func (t *tree) Message(id NodeID) string { return t.strs[t.MessageID(id)] } +func (t *tree) NameID(id NodeID) int32 { return t.nodes[id].nameID } +func (t *tree) Name(id NodeID) string { return t.strs[t.NameID(id)] } +func (t *tree) IsType(id NodeID, typ NodeType) bool { return t.Type(id) == typ } +func (t *tree) IsNamed(id NodeID, nameID int32) bool { + return t.Type(id) == NodeType_Node && t.NameID(id) == nameID +} +func (t *tree) Location(cursor int) Location { + t.ensurePosView() + return t.posView.LocationAt(cursor) +} +func (t *tree) Span(id NodeID) Span { + t.ensurePosView() + n := &t.nodes[id] + return Span{ + Start: t.posView.LocationAt(n.start), + End: t.posView.LocationAt(n.end), + } +} +func (t *tree) CursorU16(cursor int) int { + t.ensurePosView() + return t.posView.CursorU16(cursor) +} +func (t *tree) ensurePosView() { + if t.posView == nil { + t.posView = newPosIndex(t.input) + } +} +func (t *tree) Children(id NodeID) []NodeID { + n := &t.nodes[id] + if n.childID == -1 { + return nil + } + if n.typ == NodeType_Node || n.typ == NodeType_Error { + return []NodeID{NodeID(n.childID)} + } + if n.typ == NodeType_Sequence { + cr := t.childRanges[n.childID] + return t.children[cr.start:cr.end] + } + return nil +} +func (t *tree) Child(id NodeID) (NodeID, bool) { + childID := t.nodes[id].childID + if childID == -1 { + return 0, false + } + return NodeID(childID), true +} +func (t *tree) Copy() Tree { + + if t == nil { + return nil + } + + var ( + nodes = make([]node, len(t.nodes)) + children = make([]NodeID, len(t.children)) + childRanges = make([]struct{ start, end int32 }, len(t.childRanges)) + ) + copy(nodes, t.nodes) + copy(children, t.children) + copy(childRanges, t.childRanges) + + return &tree{ + nodes: nodes, + children: children, + childRanges: childRanges, + strs: t.strs, + input: t.input, + root: t.root, + } +} +func (t *tree) AddString(start, end int) NodeID { + id := NodeID(len(t.nodes)) + t.nodes = append(t.nodes, node{ + typ: NodeType_String, + start: start, + end: end, + nameID: -1, + childID: -1, + messageID: -1, + }) + return id +} +func (t *tree) AddSequence(children []NodeID, start, end int) NodeID { + id := NodeID(len(t.nodes)) + childRangeID := int32(-1) + if len(children) > 0 { + childRangeID = int32(len(t.childRanges)) + childStart := int32(len(t.children)) + t.children = append(t.children, children...) + childEnd := int32(len(t.children)) + t.childRanges = append(t.childRanges, struct{ start, end int32 }{childStart, childEnd}) + } + t.nodes = append(t.nodes, node{ + typ: NodeType_Sequence, + start: start, + end: end, + childID: childRangeID, + nameID: -1, + messageID: -1, + }) + return id +} +func (t *tree) AddNode(nameID int32, child NodeID, start, end int) NodeID { + id := NodeID(len(t.nodes)) + t.nodes = append(t.nodes, node{ + typ: NodeType_Node, + start: start, + end: end, + nameID: nameID, + childID: int32(child), + messageID: -1, + }) + return id +} +func (t *tree) AddNamedString(nameID int32, start, end int) NodeID { + stringID := NodeID(len(t.nodes)) + t.nodes = append(t.nodes, node{ + typ: NodeType_String, + start: start, + end: end, + nameID: -1, + childID: -1, + messageID: -1, + }, node{ + typ: NodeType_Node, + start: start, + end: end, + nameID: nameID, + childID: int32(stringID), + messageID: -1, + }) + return stringID + 1 +} +func (t *tree) AddError(labelID, messageID int32, start, end int) NodeID { + id := NodeID(len(t.nodes)) + t.nodes = append(t.nodes, node{ + typ: NodeType_Error, + start: start, + end: end, + nameID: labelID, + childID: -1, + messageID: messageID, + }) + return id +} +func (t *tree) AddErrorWithChild(labelID, messageID int32, childID NodeID, start, end int) NodeID { + id := NodeID(len(t.nodes)) + t.nodes = append(t.nodes, node{ + typ: NodeType_Error, + start: start, + end: end, + nameID: labelID, + childID: int32(childID), + messageID: messageID, + }) + return id +} +func (t *tree) Visit(id NodeID, fn func(NodeID) bool) { + if !fn(id) { + return + } + switch t.nodes[id].typ { + case NodeType_Sequence: + for _, child := range t.Children(id) { + t.Visit(child, fn) + } + case NodeType_Node, NodeType_Error: + if child, ok := t.Child(id); ok { + t.Visit(child, fn) + } + } +} +func (t *tree) Text(id NodeID) string { + n := &t.nodes[id] + + switch n.typ { + case NodeType_String: + return string(t.input[n.start:n.end]) + + case NodeType_Sequence: + var b strings.Builder + for _, childID := range t.Children(id) { + b.WriteString(t.Text(childID)) + } + return b.String() + + case NodeType_Node: + if child, ok := t.Child(id); ok { + return t.Text(child) + } + return "" + + case NodeType_Error: + if child, ok := t.Child(id); ok { + return t.Text(child) + } + return fmt.Sprintf("error[%s]", t.Name(id)) + default: + panic(fmt.Sprintf("Unknown node type: %T", n.typ)) + } +} +func (t *tree) Pretty(id NodeID) string { + vi := newPrettyPrinter(t, t.input, func(input string, _ FormatToken) string { + return input + }) + vi.visit(id) + return vi.output.String() +} +func (t *tree) Highlight(id NodeID) string { + vi := newPrettyPrinter(t, t.input, func(input string, token FormatToken) string { + return treePrinterTheme[token] + input + treePrinterTheme[FormatToken_None] + }) + vi.visit(id) + return vi.output.String() +} + +var treePrinterTheme = map[FormatToken]string{ + FormatToken_None: "\033[0m", + FormatToken_Range: "\033[1;31;5;228m", + FormatToken_Literal: "\033[1;38;5;245m", + FormatToken_Error: "\033[1;38;5;127m", +} + +type prettyPrinter struct { + input []byte + tree *tree + *treePrinter[FormatToken] +} + +func newPrettyPrinter(tree *tree, input []byte, format FormatFunc[FormatToken]) *prettyPrinter { + return &prettyPrinter{ + tree: tree, + input: input, + treePrinter: newTreePrinter(format), + } +} +func (vi *prettyPrinter) visit(id NodeID) { + n := &vi.tree.nodes[id] + s := vi.tree.Span(id) + + switch n.typ { + case NodeType_String: + text := string(vi.input[n.start:n.end]) + escaped := strconv.Quote(text) + vi.write(vi.format(escaped, FormatToken_Literal)) + vi.write(vi.format(fmt.Sprintf(" (%s)", s.String()), FormatToken_Range)) + + case NodeType_Sequence: + children := vi.tree.Children(id) + seq := fmt.Sprintf("Sequence<%d> (%s)", len(children), s.String()) + vi.writel(vi.format(seq, FormatToken_Range)) + for i, child := range children { + switch { + case i == len(children)-1: + vi.pwrite("└── ") + vi.indent(" ") + vi.visit(child) + vi.unindent() + default: + vi.pwrite("├── ") + vi.indent("│ ") + vi.visit(child) + vi.unindent() + vi.write("\n") + } + } + + case NodeType_Node: + name := vi.tree.Name(id) + rgst := fmt.Sprintf(" (%s)", s.String()) + vi.write(vi.format(name, FormatToken_Literal)) + vi.writel(vi.format(rgst, FormatToken_Range)) + vi.pwrite("└── ") + vi.indent(" ") + if child, ok := vi.tree.Child(id); ok { + vi.visit(child) + } + vi.unindent() + + case NodeType_Error: + label := vi.tree.Name(id) + vi.write(vi.format(fmt.Sprintf("Error<%s>", label), FormatToken_Error)) + vi.write(vi.format(fmt.Sprintf(" (%s)", s.String()), FormatToken_Range)) + + if child, ok := vi.tree.Child(id); ok { + vi.writel("") + vi.pwrite("└── ") + vi.indent(" ") + vi.visit(child) + vi.unindent() + } + } +} + +type FormatFunc[T any] func(input string, token T) string +type treePrinter[T any] struct { + padStr *[]string + output *strings.Builder + format FormatFunc[T] +} + +func newTreePrinter[T any](format FormatFunc[T]) *treePrinter[T] { + return &treePrinter[T]{ + padStr: &[]string{}, + output: &strings.Builder{}, + format: format, + } +} +func (tp *treePrinter[T]) indent(s string) { + *tp.padStr = append(*tp.padStr, s) +} +func (tp *treePrinter[T]) unindent() { + index := len(*tp.padStr) - 1 + *tp.padStr = (*tp.padStr)[:index] +} +func (tp *treePrinter[T]) padding() { + for _, item := range *tp.padStr { + tp.write(item) + } +} +func (tp *treePrinter[T]) writel(s string) { + tp.write(s) + tp.output.WriteRune('\n') +} +func (tp *treePrinter[T]) pwritel(s string) { + tp.pwrite(s) + tp.output.WriteRune('\n') +} +func (tp *treePrinter[T]) write(s string) { + tp.output.WriteString(s) +} +func (tp *treePrinter[T]) pwrite(s string) { + tp.padding() + tp.write(s) +} + +var literalSanitizer = strings.NewReplacer( + `"`, `\"`, + `\`, `\\`, + string('\n'), `\n`, + string('\r'), `\r`, + string('\t'), `\t`, +) + +func escapeLiteral(s string) string { + return literalSanitizer.Replace(s) +} + +type FileLoadError struct { + Path string + Err error +} + +func (e *FileLoadError) Error() string { + return fmt.Sprintf("failed to load grammar file %s: %v", e.Path, e.Err) +} +func (e *FileLoadError) Unwrap() error { + return e.Err +} + +type ParsingError struct { + Message string + Label string + Start, End int + Expected []ErrHint + FFPPC int +} +type ErrHintType uint8 + +const ( + ErrHintType_Unknown ErrHintType = iota + ErrHintType_EOF + ErrHintType_Char + ErrHintType_Range +) + +type ErrHint struct { + Type ErrHintType + Char rune + Range [2]rune +} + +func (eh *ErrHint) eq(oh *ErrHint) bool { + if eh.Type != oh.Type { + return false + } + switch eh.Type { + case ErrHintType_Char: + return eh.Char == oh.Char + case ErrHintType_Range: + return eh.Range[0] == oh.Range[0] && eh.Range[1] == oh.Range[1] + case ErrHintType_EOF: + return true + } + return false +} +func (eh ErrHint) String() string { + switch eh.Type { + case ErrHintType_EOF: + return "end of input" + case ErrHintType_Char: + return formatChar(eh.Char) + case ErrHintType_Range: + return fmt.Sprintf("'%c'-'%c'", eh.Range[0], eh.Range[1]) + default: + return "?" + } +} +func formatChar(c rune) string { + switch { + case c == 0: + return "end of input" + case c == '\n': + return "newline" + case c == '\r': + return "carriage return" + case c == '\t': + return "tab" + case c == ' ': + return "space" + case c < 32 || c == 127: + + return fmt.Sprintf("'\\x%02x'", c) + default: + return fmt.Sprintf("'%c'", c) + } +} +func FormatExpectedMessage(hints []ErrHint, input []byte, pos int) string { + if len(hints) == 0 { + return "" + } + parts := make([]string, len(hints)) + for i, h := range hints { + parts[i] = h.String() + } + got := "end of input" + if pos < len(input) { + got = fmt.Sprintf("'%c'", input[pos]) + } + if len(parts) == 1 { + return fmt.Sprintf("Expected %s but got %s", parts[0], got) + } + return fmt.Sprintf("Expected %s but got %s", + joinWithOr(parts), got) +} +func joinWithOr(items []string) string { + if len(items) == 0 { + return "" + } + if len(items) == 1 { + return items[0] + } + if len(items) == 2 { + return items[0] + " or " + items[1] + } + + result := "" + for i, item := range items { + if i == len(items)-1 { + result += "or " + item + } else { + result += item + ", " + } + } + return result +} +func (e ParsingError) Error() string { + message := e.Label + if e.Message != "" { + message = e.Message + } + + span := fmt.Sprintf("%d", e.Start+1) + if e.Start != e.End { + span += fmt.Sprintf("..%d", e.End+1) + } + return fmt.Sprintf("%s @ %s", message, span) +} +func isthrown(err error) bool { + _, ok := err.(ParsingError) + return ok +} +func (s DiagnosticSeverity) String() string { + switch s { + case DiagnosticError: + return "error" + case DiagnosticWarning: + return "warning" + case DiagnosticInfo: + return "info" + case DiagnosticHint: + return "hint" + default: + return "unknown" + } +} +func (d Diagnostic) String() string { + return fmt.Sprintf("[%s] %s: %s", d.Severity, d.Code, d.Message) +} +func (d Diagnostic) FormatCLI() string { + loc := d.Location.Span.Start + return fmt.Sprintf( + "%s:%d:%d: %s: %s [%s]", + d.FilePath, + loc.Line, + loc.Column, + d.Severity, + d.Message, + d.Code, + ) +} + +type frameType int8 + +const ( + frameType_Backtracking frameType = iota + frameType_Call + frameType_Capture + frameType_LRCall +) +const lrResultLeftRec = -1 + +type frame struct { + cursor int + + pc uint32 + + capId uint32 + + nodesStart uint32 + nodesEnd uint32 + + t frameType + + predicate bool + + lrIdx uint32 +} +type lrFrameData struct { + address int + + precedence int + + result int + + committedEnd uint32 +} +type stack struct { + frames []frame + nodeArena []NodeID + nodes []NodeID + tree *tree + + lrData []lrFrameData +} + +func (s *stack) lr(f *frame) *lrFrameData { + return &s.lrData[f.lrIdx-1] +} +func (s *stack) pushLR(data lrFrameData) uint32 { + s.lrData = append(s.lrData, data) + return uint32(len(s.lrData)) +} +func (s *stack) push(f frame) { + f.nodesStart = uint32(len(s.nodeArena)) + f.nodesEnd = f.nodesStart + s.frames = append(s.frames, f) +} +func (s *stack) pop() frame { + idx := len(s.frames) - 1 + f := s.frames[idx] + s.frames = s.frames[:idx] + return f +} +func (s *stack) top() *frame { + return &s.frames[len(s.frames)-1] +} +func (s *stack) len() int { + return len(s.frames) +} +func (s *stack) frameNodes(f *frame) []NodeID { + return s.nodeArena[f.nodesStart:f.nodesEnd] +} +func (s *stack) capture(nodes ...NodeID) { + if len(nodes) == 0 { + return + } + n := len(s.frames) + if n > 0 { + + s.nodeArena = append(s.nodeArena, nodes...) + s.frames[n-1].nodesEnd = uint32(len(s.nodeArena)) + return + } + s.nodes = append(s.nodes, nodes...) +} +func (s *stack) popAndCapture() frame { + idx := len(s.frames) - 1 + f := s.frames[idx] + s.frames = s.frames[:idx] + + if f.nodesStart != f.nodesEnd { + if idx > 0 { + + s.frames[idx-1].nodesEnd = f.nodesEnd + } else { + + s.nodes = append(s.nodes, s.nodeArena[f.nodesStart:f.nodesEnd]...) + } + } + return f +} +func (s *stack) collectCaptures() { + n := len(s.frames) + if n == 0 { + return + } + f := &s.frames[n-1] + if f.nodesEnd > f.nodesStart { + if n == 1 { + s.nodes = append(s.nodes, s.nodeArena[f.nodesStart:f.nodesEnd]...) + } else { + + s.frames[n-2].nodesEnd = f.nodesEnd + } + } +} +func (s *stack) truncateArena(pos uint32) { + s.nodeArena = s.nodeArena[:pos] +} +func (s *stack) reset() { + s.frames = s.frames[:0] + s.nodeArena = s.nodeArena[:0] + s.nodes = s.nodes[:0] + if s.lrData != nil { + s.lrData = s.lrData[:0] + } +} + +type charset struct { + bits [32]byte +} + +func newCharSet() *charset { return &charset{} } +func newCharsetForRune(r rune) *charset { + cs := newCharSet() + cs.add(r) + return cs +} +func newCharsetForRange(a, b rune) *charset { + cs := newCharSet() + cs.addRange(a, b) + return cs +} +func fitcs(r rune) bool { return int(r) < 0x80 && int(r)>>7 != 1 } +func (cs *charset) complement() *charset { + newCs := charset{} + for i, item := range cs.bits { + newCs.bits[i] = ^item + } + return &newCs +} +func (cs *charset) begin() int { return 0 } +func (cs *charset) end() int { return 256 } +func (cs *charset) eq(o *charset) bool { return cs.encoded() == o.encoded() } +func (cs *charset) encoded() string { return hex.EncodeToString(cs.bits[:]) } +func (cs *charset) add(r rune) { + if !fitcs(r) { + panic(fmt.Sprintf("code point `U+%X` (%c) is out of bounds", r, r)) + } + i := int(r) + cs.bits[i>>3] |= 1 << (i & 7) +} +func (cs *charset) addRange(start, end rune) { + if !fitcs(start) || !fitcs(end) || start > end { + panic(fmt.Sprintf("range out of charset bounds: `%c-%c`", start, end)) + } + for r := start; r <= end; r++ { + cs.add(r) + } +} +func (cs *charset) hasByte(i byte) bool { + + return cs.bits[i>>3]&(1<<(i&7)) != 0 +} +func (cs *charset) popcount() int { + var total int + for _, oneByte := range cs.bits { + total += bits.OnesCount16(uint16(oneByte)) + } + return total +} +func charsetMerge(a, b *charset) *charset { + out := newCharSet() + for i := 0; i < 32; i++ { + out.bits[i] = a.bits[i] | b.bits[i] + } + return out +} +func (cs *charset) precomputeExpectedSet() []expected { + var ( + ex []expected + rg bool + st int + pr int = -2 + ) + + if cs.popcount() > 100 { + return ex + } + for r := cs.begin(); r < cs.end(); r++ { + has := cs.hasByte(byte(r)) + if has { + if !rg { + rg = true + st = r + } + pr = r + } else if rg { + rg = false + addRangeToSlice(&ex, rune(st), rune(pr)) + } + } + if rg { + addRangeToSlice(&ex, rune(st), rune(pr)) + } + + return ex +} +func addRangeToSlice(ex *[]expected, start, end rune) { + if start == end { + *ex = append(*ex, expected{a: start}) + } else if end == start+1 { + *ex = append(*ex, expected{a: start}) + *ex = append(*ex, expected{a: end}) + } else { + *ex = append(*ex, expected{a: start, b: end}) + } +} +func (cs *charset) String() string { + var ( + s strings.Builder + rg bool + st int + pr int = -2 + ) + s.WriteString("[") + + for r := cs.begin(); r < cs.end(); r++ { + has := cs.hasByte(byte(r)) + if has { + if !rg { + rg = true + st = r + } + pr = r + } else if rg { + rg = false + addRangeToStr(&s, rune(st), rune(pr)) + } + } + if rg { + addRangeToStr(&s, rune(st), rune(pr)) + } + + s.WriteString("]") + return s.String() +} +func addRangeToStr(s *strings.Builder, start, end rune) { + if start == end { + s.WriteString(escapeLiteral(string(start))) + } else if end == start+1 { + s.WriteString(escapeLiteral(string(start))) + s.WriteString(escapeLiteral(string(end))) + } else { + s.WriteString(escapeLiteral(string(start))) + s.WriteString("..") + s.WriteString(escapeLiteral(string(end))) + } +} + +type Bytecode struct { + code []byte + strs []string + sets []charset + sexp [][]expected + smap map[string]int + rxps map[int]int + rxbs bitset512 + srcm *SourceMap +} +type lrMemoKey struct { + address int + cursor int +} +type lrMemoEntry struct { + cursor int + + bound int + + precedence int + + captures []NodeID +} + +func (b *Bytecode) CompileErrorLabels(labels map[string]string) map[int]int { + if len(labels) == 0 { + return nil + } + result := make(map[int]int, len(labels)) + for label, message := range labels { + labelID, ok := b.smap[label] + if !ok { + continue + } + messageID, ok := b.smap[message] + if !ok { + messageID = len(b.strs) + b.strs = append(b.strs, message) + b.smap[message] = messageID + } + result[labelID] = messageID + } + return result +} + +type bitset512 [8]uint64 + +func (b *bitset512) Set(id int) { b[id>>6] |= 1 << (id & 63) } +func (b *bitset512) Has(id int) bool { return b[id>>6]&(1<<(id&63)) != 0 } + +type expected struct { + a, b rune +} + +const expectedLimit = 20 + +type expectedInfo struct { + cur int + arr [expectedLimit]expected +} + +func (e *expectedInfo) add(s expected) { + if e.cur == expectedLimit { + return + } + if s.b == 0 { + switch s.a { + case 0, ' ', '\n', '\r', '\t': + return + } + } + for i := 0; i < e.cur; i++ { + if e.arr[i] == s { + return + } + } + e.arr[e.cur] = s + e.cur++ +} +func (e *expectedInfo) clear() { + e.cur = 0 +} + +type virtualMachine struct { + ffp int + ffpPC int + stack *stack + bytecode *Bytecode + predicate bool + expected *expectedInfo + showFails bool + errLabels map[int]int + capOffsetId int + capOffsetStart int + lrmemo map[lrMemoKey]*lrMemoEntry +} + +const ( + opHalt byte = iota + opAny + opChar + opRange + opFail + opFailTwice + opChoice + opChoicePred + opCapCommit + opCapPartialCommit + opCapBackCommit + opCall + opCapReturn + opJump + opThrow + opCapBegin + opCapEnd + opSet + opSpan + opCapTerm + opCapNonTerm + opCommit + opBackCommit + opPartialCommit + opReturn + opCapTermBeginOffset + opCapNonTermBeginOffset + opCapEndOffset + opChar32 + opRange32 + opCallLR + opReturnLR + opCapReturnLR +) + +var opNames = map[byte]string{ + opHalt: "halt", + opAny: "any", + opChar: "char", + opChar32: "char32", + opRange: "range", + opRange32: "range32", + opSet: "set", + opSpan: "span", + opFail: "fail", + opFailTwice: "fail_twice", + opChoice: "choice", + opChoicePred: "choice_pred", + opCommit: "commit", + opPartialCommit: "partial_commit", + opBackCommit: "back_commit", + opCapCommit: "cap_commit", + opCapBackCommit: "cap_back_commit", + opCapPartialCommit: "cap_partial_commit", + opCapReturn: "cap_return", + opCall: "call", + opReturn: "return", + opJump: "jump", + opThrow: "throw", + opCapBegin: "cap_begin", + opCapEnd: "cap_end", + opCapTerm: "cap_term", + opCapNonTerm: "cap_non_term", + opCapTermBeginOffset: "cap_term_begin_offset", + opCapNonTermBeginOffset: "cap_non_term_begin_offset", + opCapEndOffset: "cap_end_offset", + opCallLR: "call_lr", + opReturnLR: "return_lr", + opCapReturnLR: "cap_return_lr", +} +var ( + opAnySizeInBytes = 1 + + opCharSizeInBytes = 3 + + opChar32SizeInBytes = 5 + + opRangeSizeInBytes = 5 + + opRange32SizeInBytes = 9 + + opSetSizeInBytes = 3 + opSpanSizeInBytes = 3 + + opChoiceSizeInBytes = 3 + opCommitSizeInBytes = 3 + opFailSizeInBytes = 1 + + opCallSizeInBytes = 4 + opCallLRSizeInBytes = 4 + + opReturnSizeInBytes = 1 + + opReturnLRSizeInBytes = 1 + opCapReturnLRSizeInBytes = 1 + opJumpSizeInBytes = 3 + opThrowSizeInBytes = 3 + opHaltSizeInBytes = 1 + opCapBeginSizeInBytes = 3 + opCapEndSizeInBytes = 1 + opCapTermSizeInBytes = 3 + opCapNonTermSizeInBytes = 5 + opCapTermBeginOffsetSizeInBytes = 1 + opCapNonTermBeginOffsetSizeInBytes = 3 + opCapEndOffsetSizeInBytes = 1 +) + +func NewVirtualMachine(bytecode *Bytecode) *virtualMachine { + tr := &tree{ + nodes: make([]node, 0, 256), + children: make([]NodeID, 0, 512), + childRanges: make([]struct{ start, end int32 }, 0, 256), + } + stk := &stack{ + frames: make([]frame, 0, 256), + nodeArena: make([]NodeID, 0, 256), + nodes: make([]NodeID, 0, 256), + tree: tr, + } + stk.tree.bindStrings(bytecode.strs) + vm := &virtualMachine{ + stack: stk, + bytecode: bytecode, + ffp: -1, + } + return vm +} +func (vm *virtualMachine) SetShowFails(showFails bool) { + if showFails { + vm.expected = &expectedInfo{} + vm.showFails = true + return + } + vm.showFails = false +} +func (vm *virtualMachine) SetLabelMessages(labels map[int]int) { + vm.errLabels = labels +} +func (vm *virtualMachine) SourceMap() *SourceMap { + return vm.bytecode.srcm +} +func (vm *virtualMachine) Match(data []byte) (Tree, int, error) { + return vm.MatchRule(data, 0) +} +func (vm *virtualMachine) MatchRule(data []byte, ruleAddress int) (Tree, int, error) { + + stack := vm.stack + code := vm.bytecode.code + sets := vm.bytecode.sets + ilen := len(data) + cursor := 0 + pc := 0 + + stack.reset() + stack.tree.reset() + stack.tree.bindInput(data) + vm.ffp = -1 + vm.ffpPC = 0 + if vm.showFails { + vm.expected.clear() + } + if vm.lrmemo != nil { + for k := range vm.lrmemo { + delete(vm.lrmemo, k) + } + } + + if ruleAddress > 0 { + stack.push(vm.mkCallFrame(opCallSizeInBytes)) + pc = ruleAddress + } +code: + for { + op := code[pc] + + switch op { + case opHalt: + + if len(stack.nodes) > 0 { + idx := len(stack.nodes) - 1 + nid := stack.nodes[idx] + stack.tree.SetRoot(nid) + } + return stack.tree, cursor, nil + + case opAny: + if cursor >= ilen { + goto fail + } + _, s := decodeRune(data, cursor) + cursor += s + pc += opAnySizeInBytes + + case opChar: + e := rune(decodeU16(code, pc+1)) + if cursor >= ilen { + goto fail + } + c, s := decodeRune(data, cursor) + if c != e { + if vm.showFails { + vm.updateExpected(cursor, expected{a: e}) + } + goto fail + } + cursor += s + pc += opCharSizeInBytes + + case opChar32: + e := rune(decodeU32(code, pc+1)) + if cursor >= ilen { + goto fail + } + c, s := decodeRune(data, cursor) + if c != e { + if vm.showFails { + vm.updateExpected(cursor, expected{a: e}) + } + goto fail + } + cursor += s + pc += opChar32SizeInBytes + + case opRange: + if cursor >= ilen { + goto fail + } + c, s := decodeRune(data, cursor) + a := rune(decodeU16(code, pc+1)) + b := rune(decodeU16(code, pc+3)) + if c < a || c > b { + if vm.showFails { + vm.updateExpected(cursor, expected{a: a, b: b}) + } + goto fail + } + cursor += s + pc += opRangeSizeInBytes + + case opRange32: + if cursor >= ilen { + goto fail + } + c, s := decodeRune(data, cursor) + a := rune(decodeU32(code, pc+1)) + b := rune(decodeU32(code, pc+5)) + if c < a || c > b { + if vm.showFails { + vm.updateExpected(cursor, expected{a: a, b: b}) + } + goto fail + } + cursor += s + pc += opRange32SizeInBytes + + case opSet: + if cursor >= ilen { + goto fail + } + c := data[cursor] + i := decodeU16(code, pc+1) + if !sets[i].hasByte(c) { + if vm.showFails { + vm.updateSetExpected(cursor, i) + } + goto fail + } + cursor++ + pc += opSetSizeInBytes + + case opSpan: + sid := decodeU16(code, pc+1) + set := sets[sid] + for cursor < ilen { + c := data[cursor] + if set.hasByte(c) { + cursor++ + continue + } + break + } + pc += opSetSizeInBytes + + case opFail: + goto fail + + case opFailTwice: + stack.pop() + goto fail + + case opChoice: + lb := int(decodeU16(code, pc+1)) + stack.push(mkBacktrackFrame(lb, cursor)) + pc += opChoiceSizeInBytes + + case opChoicePred: + lb := int(decodeU16(code, pc+1)) + stack.push(mkBacktrackPredFrame(lb, cursor)) + pc += opChoiceSizeInBytes + vm.predicate = true + + case opCommit: + stack.pop() + pc = int(decodeU16(code, pc+1)) + + case opBackCommit: + cursor = stack.pop().cursor + pc = int(decodeU16(code, pc+1)) + + case opPartialCommit: + pc = int(decodeU16(code, pc+1)) + stack.top().cursor = cursor + + case opCall: + stack.push(vm.mkCallFrame(pc + opCallSizeInBytes)) + pc = int(decodeU16(code, pc+1)) + + case opCallLR: + var failed bool + if pc, cursor, failed = vm.doCallLR(stack, pc, cursor); failed { + goto fail + } + + case opReturn: + f := stack.pop() + pc = int(f.pc) + + case opReturnLR: + pc, cursor = vm.doReturnLR(stack, cursor) + + case opJump: + pc = int(decodeU16(code, pc+1)) + + case opThrow: + if vm.predicate { + pc += opThrowSizeInBytes + goto fail + } + lb := int(decodeU16(code, pc+1)) + if addr, ok := vm.bytecode.rxps[lb]; ok { + stack.push(vm.mkCallFrame(pc + opThrowSizeInBytes)) + pc = addr + continue + } + return nil, cursor, vm.mkErr(data, lb, cursor, vm.ffp) + + case opCapBegin: + id := int(decodeU16(code, pc+1)) + stack.push(vm.mkCaptureFrame(id, cursor)) + pc += opCapBeginSizeInBytes + + case opCapEnd: + f := stack.pop() + nodes := stack.frameNodes(&f) + stack.truncateArena(f.nodesStart) + vm.newNode(cursor, f, nodes) + pc += opCapEndSizeInBytes + + case opCapTerm: + offset := int(decodeU16(code, pc+1)) + if offset > 0 { + begin := cursor - offset + nodeID := stack.tree.AddString(begin, cursor) + stack.capture(nodeID) + } + pc += opCapTermSizeInBytes + + case opCapNonTerm: + id := int(decodeU16(code, pc+1)) + offset := int(decodeU16(code, pc+3)) + if offset > 0 { + begin := cursor - offset + named := stack.tree.AddNamedString(int32(id), begin, cursor) + stack.capture(named) + } + pc += opCapNonTermSizeInBytes + + case opCapTermBeginOffset: + vm.capOffsetId = -1 + vm.capOffsetStart = cursor + pc += opCapTermBeginOffsetSizeInBytes + + case opCapNonTermBeginOffset: + vm.capOffsetId = int(decodeU16(code, pc+1)) + vm.capOffsetStart = cursor + pc += opCapNonTermBeginOffsetSizeInBytes + + case opCapEndOffset: + offset := cursor - vm.capOffsetStart + pc += opCapEndOffsetSizeInBytes + if offset > 0 { + begin := cursor - offset + if vm.capOffsetId < 0 { + nodeID := stack.tree.AddString(begin, cursor) + stack.capture(nodeID) + } else { + named := stack.tree.AddNamedString(int32(vm.capOffsetId), begin, cursor) + stack.capture(named) + } + } + + case opCapCommit: + stack.popAndCapture() + pc = int(decodeU16(code, pc+1)) + + case opCapBackCommit: + f := stack.popAndCapture() + cursor = f.cursor + pc = int(decodeU16(code, pc+1)) + + case opCapPartialCommit: + pc = int(decodeU16(code, pc+1)) + top := stack.top() + top.cursor = cursor + stack.collectCaptures() + + top.nodesStart = uint32(len(stack.nodeArena)) + top.nodesEnd = top.nodesStart + + case opCapReturn: + f := stack.popAndCapture() + pc = int(f.pc) + + case opCapReturnLR: + pc, cursor = vm.doCapReturnLR(stack, cursor) + + default: + panic("NO ENTIENDO SENOR") + } + } + +fail: + if cursor > vm.ffp { + vm.ffp = cursor + vm.ffpPC = pc + } + + for stack.len() > 0 { + f := stack.pop() + stack.truncateArena(f.nodesStart) + + if f.t == frameType_Backtracking { + pc = int(f.pc) + vm.predicate = f.predicate + cursor = f.cursor + + goto code + } + if f.t == frameType_LRCall { + var gotoCode bool + if pc, cursor, gotoCode = vm.doFailLR(stack, &f); gotoCode { + goto code + } + } + } + + if len(stack.nodes) > 0 { + idx := len(stack.nodes) - 1 + nid := stack.nodes[idx] + stack.tree.SetRoot(nid) + } + return stack.tree, cursor, vm.mkErr(data, 0, cursor, vm.ffp) +} +func (vm *virtualMachine) doCallLR(stack *stack, pc, cursor int) (int, int, bool) { + var ( + code = vm.bytecode.code + addr = int(decodeU16(code, pc+1)) + prec = int(code[pc+3]) + key = lrMemoKey{address: addr, cursor: cursor} + ) + if vm.lrmemo == nil { + vm.lrmemo = make(map[lrMemoKey]*lrMemoEntry) + } + entry := vm.lrmemo[key] + + if entry == nil { + + vm.lrmemo[key] = &lrMemoEntry{ + cursor: lrResultLeftRec, + bound: 0, + precedence: prec, + captures: nil, + } + captureStart := uint32(len(stack.nodeArena)) + lrIdx := stack.pushLR(lrFrameData{ + address: addr, + precedence: prec, + result: lrResultLeftRec, + committedEnd: captureStart, + }) + stack.push(frame{ + t: frameType_LRCall, + pc: uint32(pc + opCallLRSizeInBytes), + cursor: cursor, + lrIdx: lrIdx, + nodesStart: captureStart, + nodesEnd: captureStart, + }) + return addr, cursor, false + } + + if entry.cursor == lrResultLeftRec || prec < entry.precedence { + + return 0, 0, true + } + + for _, nodeID := range entry.captures { + stack.capture(nodeID) + } + return pc + opCallLRSizeInBytes, entry.cursor, false +} +func (vm *virtualMachine) doReturnLR(stack *stack, cursor int) (int, int) { + var ( + f = stack.top() + lr = stack.lr(f) + key = lrMemoKey{address: lr.address, cursor: f.cursor} + entry = vm.lrmemo[key] + ) + if lr.result == lrResultLeftRec || cursor > lr.result { + + entry.cursor = cursor + entry.bound++ + entry.precedence = lr.precedence + lr.result = cursor + return lr.address, f.cursor + } + + stack.pop() + var ( + newCursor = lr.result + newPC = int(f.pc) + ) + delete(vm.lrmemo, key) + return newPC, newCursor +} +func (vm *virtualMachine) doCapReturnLR(stack *stack, cursor int) (int, int) { + var ( + f = stack.top() + lr = stack.lr(f) + key = lrMemoKey{address: lr.address, cursor: f.cursor} + entry = vm.lrmemo[key] + ) + if lr.result == lrResultLeftRec || cursor > lr.result { + + currentCaptures := make([]NodeID, f.nodesEnd-f.nodesStart) + copy(currentCaptures, stack.nodeArena[f.nodesStart:f.nodesEnd]) + + entry.captures = currentCaptures + entry.cursor = cursor + entry.bound++ + entry.precedence = lr.precedence + + stack.truncateArena(f.nodesStart) + + lr.result = cursor + lr.committedEnd = f.nodesStart + f.nodesEnd = f.nodesStart + + return lr.address, f.cursor + } + + stack.pop() + stack.truncateArena(f.nodesStart) + + for _, nodeID := range entry.captures { + stack.capture(nodeID) + } + var ( + newCursor = lr.result + newPC = int(f.pc) + ) + delete(vm.lrmemo, key) + return newPC, newCursor +} +func (vm *virtualMachine) doFailLR(stack *stack, f *frame) (int, int, bool) { + var ( + lr = stack.lr(f) + key = lrMemoKey{address: lr.address, cursor: f.cursor} + entry = vm.lrmemo[key] + ) + if lr.result == lrResultLeftRec { + + delete(vm.lrmemo, key) + return 0, 0, false + } + if lr.result > 0 && entry != nil { + + for _, nodeID := range entry.captures { + stack.capture(nodeID) + } + delete(vm.lrmemo, key) + return int(f.pc), lr.result, true + } + + delete(vm.lrmemo, key) + return 0, 0, false +} +func mkBacktrackFrame(pc, cursor int) frame { + return frame{ + t: frameType_Backtracking, + pc: uint32(pc), + cursor: cursor, + } +} +func mkBacktrackPredFrame(pc, cursor int) frame { + f := mkBacktrackFrame(pc, cursor) + f.predicate = true + return f +} +func (vm *virtualMachine) mkCaptureFrame(id, cursor int) frame { + return frame{ + t: frameType_Capture, + capId: uint32(id), + cursor: cursor, + } +} +func (vm *virtualMachine) mkCallFrame(pc int) frame { + return frame{t: frameType_Call, pc: uint32(pc)} +} +func (vm *virtualMachine) newNode(cursor int, f frame, nodes []NodeID) { + var ( + nodeID NodeID + hasNode = false + isrxp = vm.bytecode.rxbs.Has(int(f.capId)) + capId = int32(f.capId) + start = f.cursor + end = cursor + ) + switch len(nodes) { + case 0: + if cursor-f.cursor > 0 { + nodeID = vm.stack.tree.AddString(start, end) + hasNode = true + } else if !isrxp { + + return + } + + case 1: + nodeID = nodes[0] + hasNode = true + default: + nodeID = vm.stack.tree.AddSequence(nodes, start, end) + hasNode = true + } + + if isrxp { + msgID, ok := vm.errLabels[int(f.capId)] + if !ok { + msgID = int(f.capId) + } + var errNode NodeID + if hasNode { + errNode = vm.stack.tree.AddErrorWithChild(capId, int32(msgID), nodeID, start, end) + } else { + errNode = vm.stack.tree.AddError(capId, int32(msgID), start, end) + } + vm.stack.capture(errNode) + return + } + + if !hasNode { + return + } + + if f.capId == 0 { + vm.stack.capture(nodeID) + return + } + + named := vm.stack.tree.AddNode(capId, nodeID, start, end) + vm.stack.capture(named) +} +func (vm *virtualMachine) updateExpected(cursor int, s expected) { + shouldClear := cursor > vm.ffp + shouldAdd := cursor >= vm.ffp + + if shouldClear { + vm.expected.clear() + } + if shouldAdd { + vm.expected.add(s) + } +} +func (vm *virtualMachine) updateSetExpected(cursor int, sid uint16) { + shouldClear := cursor > vm.ffp + shouldAdd := cursor >= vm.ffp + + if shouldClear { + vm.expected.clear() + } + if shouldAdd { + for i, item := range vm.bytecode.sexp[sid] { + vm.expected.add(item) + if i > expectedLimit-1 { + break + } + } + } +} +func (vm *virtualMachine) mkErr(data []byte, errLabelID int, cursor, errCursor int) error { + + var ( + isEof bool + message strings.Builder + c rune + ) + if cursor >= len(data) { + isEof = true + } else { + c, _ = decodeRune(data, cursor) + } + if msgID, ok := vm.errLabels[errLabelID]; ok { + + message.WriteString(vm.bytecode.strs[msgID]) + } else { + + if errLabelID > 0 { + errLabel := vm.bytecode.strs[errLabelID] + message.WriteRune('[') + message.WriteString(errLabel) + message.WriteRune(']') + message.WriteRune(' ') + } + + if vm.showFails && vm.expected.cur > 0 { + message.WriteString("Expected ") + for i := 0; i < vm.expected.cur; i++ { + e := vm.expected.arr[i] + message.WriteRune('\'') + message.WriteRune(e.a) + if e.b != 0 { + message.WriteRune('-') + message.WriteRune(e.b) + } + message.WriteRune('\'') + if i < vm.expected.cur-1 { + message.WriteString(", ") + } + } + message.WriteString(" but got ") + } else { + message.WriteString("Unexpected ") + } + if isEof { + message.WriteString("EOF") + } else { + message.WriteRune('\'') + message.WriteRune(c) + message.WriteRune('\'') + } + } + errLabel := "" + if errLabelID > 0 { + errLabel = vm.bytecode.strs[errLabelID] + } + var expected []ErrHint + if vm.expected != nil && vm.expected.cur > 0 { + expected = make([]ErrHint, vm.expected.cur) + for i := 0; i < vm.expected.cur; i++ { + ex := vm.expected.arr[i] + switch { + case ex.a == 0: + expected[i] = ErrHint{Type: ErrHintType_EOF} + case ex.a != 0 && ex.b == 0: + expected[i] = ErrHint{ + Type: ErrHintType_Char, + Char: ex.a, + } + case ex.a != 0 && ex.b != 0: + expected[i] = ErrHint{ + Type: ErrHintType_Range, + Range: [2]rune{ex.a, ex.b}, + } + default: + expected[i] = ErrHint{Type: ErrHintType_Unknown} + } + } + } + return ParsingError{ + Message: message.String(), + Label: errLabel, + Start: cursor, + End: errCursor, + Expected: expected, + FFPPC: vm.ffpPC, + } +} +func decodeU16(code []byte, offset int) uint16 { + return uint16(code[offset]) | uint16(code[offset+1])<<8 +} +func decodeU32(code []byte, offset int) uint32 { + return uint32(code[offset]) | + uint32(code[offset+1])<<8 | + uint32(code[offset+2])<<16 | + uint32(code[offset+3])<<24 +} +func decodeRune(data []byte, offset int) (rune, int) { + if r := data[offset]; r < utf8.RuneSelf { + return rune(r), 1 + } + return utf8.DecodeRune(data[offset:]) +} From 8842020b396f642716933eeb7e811c4ff20312fd Mon Sep 17 00:00:00 2001 From: Jefferson Oliveira Date: Thu, 21 May 2026 09:17:19 -0300 Subject: [PATCH 02/11] feat(parser): add Go parser implementation and C-callable interface - Introduced a new Go module for the parser with `go.mod` file. - Added `shim.go` to expose the parser as a C-callable interface, enabling integration with Zig. - Updated `.gitignore` to include new parser files and directories. --- .gitignore | 4 +- lib/parser/go.mod | 3 + lib/parser/shim/shim.go | 141 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 lib/parser/go.mod create mode 100644 lib/parser/shim/shim.go diff --git a/.gitignore b/.gitignore index 2b42a2f..959fa1e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ zig-out/ example/ node_modules/ -.claude/ \ No newline at end of file +.claude/ +lib/parser/parser.a +lib/parser/parser.h \ No newline at end of file diff --git a/lib/parser/go.mod b/lib/parser/go.mod new file mode 100644 index 0000000..7e27a50 --- /dev/null +++ b/lib/parser/go.mod @@ -0,0 +1,3 @@ +module circ-compiler/parser + +go 1.21 diff --git a/lib/parser/shim/shim.go b/lib/parser/shim/shim.go new file mode 100644 index 0000000..c2408be --- /dev/null +++ b/lib/parser/shim/shim.go @@ -0,0 +1,141 @@ +// Package main exposes the langlang parser as a C-callable surface for +// go build -buildmode=c-archive. main() is empty; the archive is linked +// into circ-compile by Zig. +package main + +/* +#include +#include +#include +*/ +import "C" + +import ( + "runtime/cgo" + "unsafe" + + "circ-compiler/parser" +) + +type handle struct { + tree parser.Tree + + // Borrowed pointer to the source buffer; the caller (Zig) owns the + // buffer's lifetime and must keep it alive until ParserDelete. + sourcePtr *C.char + sourceLen C.int + + // Stable C-string cache so Zig can compare returned name pointers + // across calls. Keyed by string identity (not NodeID) because the + // same name (e.g. "Identifier") appears on many nodes. + nameCache map[string]*C.char +} + +//export ParserNew +func ParserNew() C.uintptr_t { + h := &handle{nameCache: make(map[string]*C.char)} + return C.uintptr_t(cgo.NewHandle(h)) +} + +//export ParserParse +func ParserParse(hid C.uintptr_t, source *C.char, length C.int) C.bool { + h := cgo.Handle(hid).Value().(*handle) + src := unsafe.Slice((*byte)(unsafe.Pointer(source)), int(length)) + p := parser.NewParser() + p.SetInput(src) + tree, err := p.Parse() + if err != nil { + return false + } + h.tree = tree + h.sourcePtr = source + h.sourceLen = length + return true +} + +//export ParserDelete +func ParserDelete(hid C.uintptr_t) { + h := cgo.Handle(hid).Value().(*handle) + for _, p := range h.nameCache { + C.free(unsafe.Pointer(p)) + } + cgo.Handle(hid).Delete() +} + +//export TreeRoot +func TreeRoot(hid C.uintptr_t, out *C.uint32_t) C.bool { + h := cgo.Handle(hid).Value().(*handle) + id, ok := h.tree.Root() + if !ok { + return false + } + *out = C.uint32_t(id) + return true +} + +//export TreeType +func TreeType(hid C.uintptr_t, id C.uint32_t) C.uint8_t { + h := cgo.Handle(hid).Value().(*handle) + return C.uint8_t(h.tree.Type(parser.NodeID(id))) +} + +//export TreeName +func TreeName(hid C.uintptr_t, id C.uint32_t) *C.char { + h := cgo.Handle(hid).Value().(*handle) + name := h.tree.Name(parser.NodeID(id)) + if cached, ok := h.nameCache[name]; ok { + return cached + } + cstr := C.CString(name) + h.nameCache[name] = cstr + return cstr +} + +//export TreeSpanStart +func TreeSpanStart(hid C.uintptr_t, id C.uint32_t) C.int { + h := cgo.Handle(hid).Value().(*handle) + return C.int(h.tree.Span(parser.NodeID(id)).Start.Cursor) +} + +//export TreeSpanEnd +func TreeSpanEnd(hid C.uintptr_t, id C.uint32_t) C.int { + h := cgo.Handle(hid).Value().(*handle) + return C.int(h.tree.Span(parser.NodeID(id)).End.Cursor) +} + +//export TreeChild +func TreeChild(hid C.uintptr_t, id C.uint32_t, out *C.uint32_t) C.bool { + h := cgo.Handle(hid).Value().(*handle) + child, ok := h.tree.Child(parser.NodeID(id)) + if !ok { + return false + } + *out = C.uint32_t(child) + return true +} + +//export TreeChildrenLen +func TreeChildrenLen(hid C.uintptr_t, id C.uint32_t) C.int { + h := cgo.Handle(hid).Value().(*handle) + return C.int(len(h.tree.Children(parser.NodeID(id)))) +} + +//export TreeChildrenAt +func TreeChildrenAt(hid C.uintptr_t, id C.uint32_t, idx C.int, out *C.uint32_t) C.bool { + h := cgo.Handle(hid).Value().(*handle) + children := h.tree.Children(parser.NodeID(id)) + if int(idx) < 0 || int(idx) >= len(children) { + return false + } + *out = C.uint32_t(children[idx]) + return true +} + +//export TreeInput +func TreeInput(hid C.uintptr_t, outLen *C.int) *C.char { + h := cgo.Handle(hid).Value().(*handle) + *outLen = h.sourceLen + return h.sourcePtr +} + +func main() {} From 626aad5de1c9da3a899312e801fbcf722d3b1335 Mon Sep 17 00:00:00 2001 From: Jefferson Oliveira Date: Thu, 21 May 2026 10:22:14 -0300 Subject: [PATCH 03/11] feat(parser): add build step for Go parser archive - Introduced a new build step in `build.zig` to create a CGo c-archive for the Go parser implementation. - Added a system command to build the Go shim as a static library, ensuring proper integration with Zig. - Updated dependencies to reflect the new build process for the parser archive. --- build.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/build.zig b/build.zig index d718733..142c20d 100644 --- a/build.zig +++ b/build.zig @@ -26,6 +26,20 @@ pub fn build(b: *std.Build) void { parser_gen.dependOn(&generate_parser_cmd.step); + const parser_archive = b.step("parser:archive", "Build CGo c-archive (lib/parser/parser.a) from Go shim"); + + const build_archive_cmd = b.addSystemCommand(&.{ + "go", + "build", + "-buildmode=c-archive", + "-o", + "parser.a", + "./shim", + }); + build_archive_cmd.setCwd(b.path("lib/parser")); + + parser_archive.dependOn(&build_archive_cmd.step); + const parser_lib = b.addLibrary(.{ .linkage = .static, .name = "parser", From f7fe54b4f2db3a9ef74fd18d7c5b5b3669a22847 Mon Sep 17 00:00:00 2001 From: Jefferson Oliveira Date: Thu, 21 May 2026 10:57:27 -0300 Subject: [PATCH 04/11] feat(parser): enhance Go parser integration and update C interface - Added environment variables for cross-compilation in `build.zig` to ensure compatibility with Zig target architecture and OS. - Updated the C interface in `CParser.zig` to include the correct header path for the Go parser. - Refactored `translate.zig` to utilize the new Go API, replacing C parser references with Go-compatible types and functions. - Streamlined the parser library linking process in the benchmark executable to use the updated static library. --- build.zig | 50 ++++++-------- lib/syntax/CParser.zig | 2 +- lib/syntax/translate.zig | 146 ++++++++++++++++++++++----------------- 3 files changed, 105 insertions(+), 93 deletions(-) diff --git a/build.zig b/build.zig index 142c20d..ea5e5f0 100644 --- a/build.zig +++ b/build.zig @@ -28,6 +28,21 @@ pub fn build(b: *std.Build) void { const parser_archive = b.step("parser:archive", "Build CGo c-archive (lib/parser/parser.a) from Go shim"); + // Tell Go to cross-compile to the same arch/OS as the Zig target. Without + // this, a Go toolchain installed for amd64 (e.g. running under Rosetta on + // Apple Silicon) emits an x86_64 archive that the arm64 linker rejects. + const goarch = switch (target.result.cpu.arch) { + .aarch64 => "arm64", + .x86_64 => "amd64", + else => @panic("unsupported CPU arch for Go c-archive build"), + }; + const goos = switch (target.result.os.tag) { + .macos => "darwin", + .linux => "linux", + .windows => "windows", + else => @panic("unsupported OS for Go c-archive build"), + }; + const build_archive_cmd = b.addSystemCommand(&.{ "go", "build", @@ -37,6 +52,9 @@ pub fn build(b: *std.Build) void { "./shim", }); build_archive_cmd.setCwd(b.path("lib/parser")); + build_archive_cmd.setEnvironmentVariable("GOARCH", goarch); + build_archive_cmd.setEnvironmentVariable("GOOS", goos); + build_archive_cmd.setEnvironmentVariable("CGO_ENABLED", "1"); parser_archive.dependOn(&build_archive_cmd.step); @@ -50,13 +68,9 @@ pub fn build(b: *std.Build) void { }), }); - parser_lib.addCSourceFile(.{ - .file = b.path("lib/parser.c"), - .flags = &.{}, - }); - - parser_lib.addIncludePath(b.path(".")); + parser_lib.addObjectFile(b.path("lib/parser/parser.a")); parser_lib.linkLibC(); + parser_lib.step.dependOn(&build_archive_cmd.step); const golden_tests = b.addTest(.{ .root_module = b.createModule(.{ @@ -1465,28 +1479,6 @@ pub fn build(b: *std.Build) void { "Optimize mode for the engine bench (default: ReleaseFast)", ) orelse .ReleaseFast; - // Parallel parser library at bench_optimize. The shared `parser_lib` is - // built at the global optimize (Debug by default), and Zig inserts UBSan - // instrumentation for C sources in Debug. When the bench links a Debug - // libparser into a ReleaseFast executable, the UBSan runtime symbols - // (`__ubsan_handle_*`) go unresolved. Easier to give the bench its own - // copy than to fight the cross-optimize-mode link. - const parser_lib_bench = b.addLibrary(.{ - .linkage = .static, - .name = "parser-bench", - .root_module = b.createModule(.{ - .root_source_file = null, - .target = target, - .optimize = bench_optimize, - }), - }); - parser_lib_bench.addCSourceFile(.{ - .file = b.path("lib/parser.c"), - .flags = &.{}, - }); - parser_lib_bench.addIncludePath(b.path(".")); - parser_lib_bench.linkLibC(); - const circuit_options_bench = b.addOptions(); circuit_options_bench.addOption(bool, "collect_metrics", true); @@ -1527,7 +1519,7 @@ pub fn build(b: *std.Build) void { }); bench_exe.addIncludePath(b.path(".")); bench_exe.addIncludePath(b.path("./lib")); - bench_exe.linkLibrary(parser_lib_bench); + bench_exe.linkLibrary(parser_lib); bench_exe.linkLibC(); const run_bench = b.addRunArtifact(bench_exe); diff --git a/lib/syntax/CParser.zig b/lib/syntax/CParser.zig index f0eca0a..0b6872d 100644 --- a/lib/syntax/CParser.zig +++ b/lib/syntax/CParser.zig @@ -1,4 +1,4 @@ pub const C_Parser = @cImport({ - @cInclude("parser.h"); + @cInclude("parser/parser.h"); @cInclude("stdlib.h"); }); diff --git a/lib/syntax/translate.zig b/lib/syntax/translate.zig index 7b35c1f..229d3b9 100644 --- a/lib/syntax/translate.zig +++ b/lib/syntax/translate.zig @@ -5,24 +5,44 @@ const C_Parser = @import("CParser.zig").C_Parser; pub const Ast = ast; +// langlang's Go API exposes NodeType discriminants as a Go iota; the +// c-archive header omits them. Mirror the values from +// lib/parser/parser.go (NodeType_String = iota, ...). +const NodeType_String: u8 = 0; +const NodeType_Sequence: u8 = 1; +const NodeType_Node: u8 = 2; +const NodeType_Error: u8 = 3; + +// Byte-offset range. Replaces C_Parser.ll_range; the Go API exposes +// richer Span (line/column) info via TreeSpanStart/TreeSpanEnd, but we +// flatten to byte offsets here so offsetToLineCol stays the source of +// truth for line/column derivation. +const Range = struct { + start: c_int, + end: c_int, +}; + const TranslationContext = struct { allocator: std.mem.Allocator, - tree: [*c]C_Parser.ll_tree, + handle: @TypeOf(C_Parser.ParserNew()), source: []const u8, file_id: u32, anonymous_counter: usize = 0, }; -fn nodeType(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) C_Parser.ll_node_type { - return C_Parser.ll_tree_type(ctx.tree, node_id); +fn nodeType(ctx: *const TranslationContext, node_id: u32) u8 { + return C_Parser.TreeType(ctx.handle, node_id); } -fn nodeName(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) []const u8 { - return std.mem.span(C_Parser.ll_tree_name(ctx.tree, node_id)); +fn nodeName(ctx: *const TranslationContext, node_id: u32) []const u8 { + return std.mem.span(C_Parser.TreeName(ctx.handle, node_id)); } -fn nodeRange(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) C_Parser.ll_range { - return C_Parser.ll_tree_range(ctx.tree, node_id); +fn nodeRange(ctx: *const TranslationContext, node_id: u32) Range { + return .{ + .start = C_Parser.TreeSpanStart(ctx.handle, node_id), + .end = C_Parser.TreeSpanEnd(ctx.handle, node_id), + }; } fn offsetToLineCol(source: []const u8, offset: usize) struct { line: u32, col: u32 } { @@ -40,7 +60,7 @@ fn offsetToLineCol(source: []const u8, offset: usize) struct { line: u32, col: u return .{ .line = line, .col = col }; } -fn nodeSpan(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) Span { +fn nodeSpan(ctx: *const TranslationContext, node_id: u32) Span { const range = nodeRange(ctx, node_id); const start: usize = @intCast(@max(range.start, 0)); const end: usize = @intCast(@max(range.end, 0)); @@ -55,35 +75,35 @@ fn nodeSpan(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) Span { }; } -fn childAt(ctx: *const TranslationContext, parent: C_Parser.ll_node_id, index: usize) !C_Parser.ll_node_id { - var child: C_Parser.ll_node_id = undefined; - if (!C_Parser.ll_tree_children_at(ctx.tree, parent, @intCast(index), &child)) { +fn childAt(ctx: *const TranslationContext, parent: u32, index: usize) !u32 { + var child: u32 = undefined; + if (!C_Parser.TreeChildrenAt(ctx.handle, parent, @intCast(index), &child)) { return error.InvalidChildIndex; } return child; } -fn childCount(ctx: *const TranslationContext, parent: C_Parser.ll_node_id) usize { - return @intCast(C_Parser.ll_tree_children_len(ctx.tree, parent)); +fn childCount(ctx: *const TranslationContext, parent: u32) usize { + return @intCast(C_Parser.TreeChildrenLen(ctx.handle, parent)); } -fn firstChild(ctx: *const TranslationContext, parent: C_Parser.ll_node_id) !C_Parser.ll_node_id { - var child: C_Parser.ll_node_id = undefined; - if (!C_Parser.ll_tree_child(ctx.tree, parent, &child)) { +fn firstChild(ctx: *const TranslationContext, parent: u32) !u32 { + var child: u32 = undefined; + if (!C_Parser.TreeChild(ctx.handle, parent, &child)) { return error.InvalidChild; } return child; } -fn expectNamedNode(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id, expected_name: []const u8) !void { - if (nodeType(ctx, node_id) != C_Parser.LL_NODE_NODE) return error.ExpectedNamedNode; +fn expectNamedNode(ctx: *const TranslationContext, node_id: u32, expected_name: []const u8) !void { + if (nodeType(ctx, node_id) != NodeType_Node) return error.ExpectedNamedNode; if (!std.mem.eql(u8, nodeName(ctx, node_id), expected_name)) return error.UnexpectedNodeName; } -fn parseIdentifier(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) !ast.Identifier { +fn parseIdentifier(ctx: *const TranslationContext, node_id: u32) !ast.Identifier { try expectNamedNode(ctx, node_id, "Identifier"); const string_node = try firstChild(ctx, node_id); - if (nodeType(ctx, string_node) != C_Parser.LL_NODE_STRING) return error.ExpectedString; + if (nodeType(ctx, string_node) != NodeType_String) return error.ExpectedString; const range = nodeRange(ctx, node_id); const start: usize = @intCast(range.start); const end: usize = @intCast(range.end); @@ -93,25 +113,25 @@ fn parseIdentifier(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) }; } -fn parseComponentTypeText(ctx: *const TranslationContext, node_id: C_Parser.ll_node_id) ![]const u8 { +fn parseComponentTypeText(ctx: *const TranslationContext, node_id: u32) ![]const u8 { try expectNamedNode(ctx, node_id, "ComponentType"); const string_node = try firstChild(ctx, node_id); - if (nodeType(ctx, string_node) == C_Parser.LL_NODE_NODE and std.mem.eql(u8, nodeName(ctx, string_node), "Identifier")) { + if (nodeType(ctx, string_node) == NodeType_Node and std.mem.eql(u8, nodeName(ctx, string_node), "Identifier")) { const ident = try parseIdentifier(ctx, string_node); return ident.text; } - if (nodeType(ctx, string_node) != C_Parser.LL_NODE_STRING) return error.ExpectedString; + if (nodeType(ctx, string_node) != NodeType_String) return error.ExpectedString; const range = nodeRange(ctx, string_node); const start: usize = @intCast(range.start); const end: usize = @intCast(range.end); return ctx.source[start..end]; } -fn parsePortRef(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror!ast.SignalSource { +fn parsePortRef(ctx: *TranslationContext, node_id: u32) anyerror!ast.SignalSource { try expectNamedNode(ctx, node_id, "PortRef"); const payload = try firstChild(ctx, node_id); switch (nodeType(ctx, payload)) { - C_Parser.LL_NODE_NODE => { + NodeType_Node => { const ident = try parseIdentifier(ctx, payload); const out_ident = ast.Identifier{ .text = "out", .span = ident.span }; return .{ @@ -122,13 +142,13 @@ fn parsePortRef(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror }, }; }, - C_Parser.LL_NODE_SEQUENCE => { + NodeType_Sequence => { const seq_len = childCount(ctx, payload); if (seq_len != 3) return error.InvalidPortReference; const left = try childAt(ctx, payload, 0); const right = try childAt(ctx, payload, 2); - if (nodeType(ctx, left) == C_Parser.LL_NODE_NODE and std.mem.eql(u8, nodeName(ctx, left), "Identifier")) { + if (nodeType(ctx, left) == NodeType_Node and std.mem.eql(u8, nodeName(ctx, left), "Identifier")) { const left_ident = try parseIdentifier(ctx, left); const right_ident = try parseIdentifier(ctx, right); return .{ @@ -140,7 +160,7 @@ fn parsePortRef(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror }; } - if (nodeType(ctx, left) == C_Parser.LL_NODE_NODE and std.mem.eql(u8, nodeName(ctx, left), "AnonDecl")) { + if (nodeType(ctx, left) == NodeType_Node and std.mem.eql(u8, nodeName(ctx, left), "AnonDecl")) { const anon_component = try parseAnonymousComponent(ctx, left); return .{ .anonymous = anon_component }; } @@ -151,22 +171,22 @@ fn parsePortRef(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror } } -fn parseBusType(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror![]ast.PortConnection { +fn parseBusType(ctx: *TranslationContext, node_id: u32) anyerror![]ast.PortConnection { try expectNamedNode(ctx, node_id, "BusType"); const seq = try firstChild(ctx, node_id); - if (nodeType(ctx, seq) != C_Parser.LL_NODE_SEQUENCE) return error.InvalidBusType; + if (nodeType(ctx, seq) != NodeType_Sequence) return error.InvalidBusType; var connections: std.ArrayList(ast.PortConnection) = .{}; const seq_len = childCount(ctx, seq); var index: usize = 0; while (index < seq_len) : (index += 1) { const child = try childAt(ctx, seq, index); - if (nodeType(ctx, child) == C_Parser.LL_NODE_NODE and std.mem.eql(u8, nodeName(ctx, child), "Identifier")) { + if (nodeType(ctx, child) == NodeType_Node and std.mem.eql(u8, nodeName(ctx, child), "Identifier")) { if (index + 2 >= seq_len) return error.InvalidBusType; const maybe_equal = try childAt(ctx, seq, index + 1); const maybe_ref = try childAt(ctx, seq, index + 2); - if (nodeType(ctx, maybe_equal) != C_Parser.LL_NODE_STRING) return error.InvalidBusType; - if (nodeType(ctx, maybe_ref) != C_Parser.LL_NODE_NODE or !std.mem.eql(u8, nodeName(ctx, maybe_ref), "PortRef")) { + if (nodeType(ctx, maybe_equal) != NodeType_String) return error.InvalidBusType; + if (nodeType(ctx, maybe_ref) != NodeType_Node or !std.mem.eql(u8, nodeName(ctx, maybe_ref), "PortRef")) { return error.InvalidBusType; } @@ -184,10 +204,10 @@ fn parseBusType(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror return connections.toOwnedSlice(ctx.allocator); } -fn parseAnonymousComponent(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) anyerror!*const ast.ComponentInstance { +fn parseAnonymousComponent(ctx: *TranslationContext, node_id: u32) anyerror!*const ast.ComponentInstance { try expectNamedNode(ctx, node_id, "AnonDecl"); const seq = try firstChild(ctx, node_id); - if (nodeType(ctx, seq) != C_Parser.LL_NODE_SEQUENCE) return error.InvalidAnonymousDeclaration; + if (nodeType(ctx, seq) != NodeType_Sequence) return error.InvalidAnonymousDeclaration; if (childCount(ctx, seq) != 2) return error.InvalidAnonymousDeclaration; const type_node = try childAt(ctx, seq, 0); @@ -208,10 +228,10 @@ fn parseAnonymousComponent(ctx: *TranslationContext, node_id: C_Parser.ll_node_i return component; } -fn parseImportDecl(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) !ast.Import { +fn parseImportDecl(ctx: *TranslationContext, node_id: u32) !ast.Import { try expectNamedNode(ctx, node_id, "ImportDecl"); const seq = try firstChild(ctx, node_id); - if (nodeType(ctx, seq) != C_Parser.LL_NODE_SEQUENCE) return error.InvalidImportDecl; + if (nodeType(ctx, seq) != NodeType_Sequence) return error.InvalidImportDecl; const seq_len = childCount(ctx, seq); if (seq_len < 4) return error.InvalidImportDecl; @@ -219,7 +239,7 @@ fn parseImportDecl(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) !ast. const path_node = try childAt(ctx, seq, 3); const alias = try parseIdentifier(ctx, alias_node); - if (nodeType(ctx, path_node) != C_Parser.LL_NODE_STRING) return error.InvalidImportDecl; + if (nodeType(ctx, path_node) != NodeType_String) return error.InvalidImportDecl; const path_range = nodeRange(ctx, path_node); const path_start: usize = @intCast(path_range.start); const path_end: usize = @intCast(path_range.end); @@ -234,19 +254,19 @@ fn parseImportDecl(ctx: *TranslationContext, node_id: C_Parser.ll_node_id) !ast. }; } -fn parseInputDecl(ctx: *TranslationContext, ident_list_node: C_Parser.ll_node_id) !ast.InputDecl { +fn parseInputDecl(ctx: *TranslationContext, ident_list_node: u32) !ast.InputDecl { try expectNamedNode(ctx, ident_list_node, "IdentList"); const payload = try firstChild(ctx, ident_list_node); var names: std.ArrayList(ast.Identifier) = .{}; - if (nodeType(ctx, payload) == C_Parser.LL_NODE_NODE) { + if (nodeType(ctx, payload) == NodeType_Node) { try names.append(ctx.allocator, try parseIdentifier(ctx, payload)); } else { const payload_len = childCount(ctx, payload); var i: usize = 0; while (i < payload_len) : (i += 1) { const c = try childAt(ctx, payload, i); - if (nodeType(ctx, c) == C_Parser.LL_NODE_NODE and std.mem.eql(u8, nodeName(ctx, c), "Identifier")) { + if (nodeType(ctx, c) == NodeType_Node and std.mem.eql(u8, nodeName(ctx, c), "Identifier")) { try names.append(ctx.allocator, try parseIdentifier(ctx, c)); } } @@ -258,7 +278,7 @@ fn parseInputDecl(ctx: *TranslationContext, ident_list_node: C_Parser.ll_node_id }; } -fn parseOutputFromBusType(ctx: *TranslationContext, instance_name: ast.Identifier, bus_node: C_Parser.ll_node_id, declaration_span: Span) !ast.OutputDecl { +fn parseOutputFromBusType(ctx: *TranslationContext, instance_name: ast.Identifier, bus_node: u32, declaration_span: Span) !ast.OutputDecl { const ports = try parseBusType(ctx, bus_node); if (ports.len == 0) return error.InvalidOutputDecl; @@ -269,7 +289,7 @@ fn parseOutputFromBusType(ctx: *TranslationContext, instance_name: ast.Identifie }; } -fn parseComponentDecl(ctx: *TranslationContext, type_name_text: []const u8, instance_name: ast.Identifier, bus_node: C_Parser.ll_node_id, declaration_span: Span) !ast.ComponentInstance { +fn parseComponentDecl(ctx: *TranslationContext, type_name_text: []const u8, instance_name: ast.Identifier, bus_node: u32, declaration_span: Span) !ast.ComponentInstance { return .{ .type_name = .{ .text = type_name_text, .span = nodeSpan(ctx, bus_node) }, .instance_name = instance_name, @@ -280,7 +300,7 @@ fn parseComponentDecl(ctx: *TranslationContext, type_name_text: []const u8, inst fn parseDeclaration( ctx: *TranslationContext, - node_id: C_Parser.ll_node_id, + node_id: u32, imports: *std.ArrayList(ast.Import), inputs: *std.ArrayList(ast.InputDecl), outputs: *std.ArrayList(ast.OutputDecl), @@ -288,7 +308,7 @@ fn parseDeclaration( ) !void { try expectNamedNode(ctx, node_id, "Declaration"); const seq = try firstChild(ctx, node_id); - if (nodeType(ctx, seq) != C_Parser.LL_NODE_SEQUENCE) return error.InvalidDeclaration; + if (nodeType(ctx, seq) != NodeType_Sequence) return error.InvalidDeclaration; const len = childCount(ctx, seq); if (len < 2) return error.InvalidDeclaration; @@ -353,8 +373,8 @@ fn parseDeclaration( } fn translateTree(ctx: *TranslationContext) !ast.File { - var root: C_Parser.ll_node_id = undefined; - if (!C_Parser.ll_tree_root(ctx.tree, &root)) return error.ParsingFailed; + var root: u32 = undefined; + if (!C_Parser.TreeRoot(ctx.handle, &root)) return error.ParsingFailed; try expectNamedNode(ctx, root, "Program"); var imports: std.ArrayList(ast.Import) = .{}; @@ -362,12 +382,12 @@ fn translateTree(ctx: *TranslationContext) !ast.File { var outputs: std.ArrayList(ast.OutputDecl) = .{}; var components: std.ArrayList(ast.ComponentInstance) = .{}; const payload = try firstChild(ctx, root); - if (nodeType(ctx, payload) == C_Parser.LL_NODE_SEQUENCE) { + if (nodeType(ctx, payload) == NodeType_Sequence) { const len = childCount(ctx, payload); var i: usize = 0; while (i < len) : (i += 1) { const child = try childAt(ctx, payload, i); - if (nodeType(ctx, child) != C_Parser.LL_NODE_NODE) continue; + if (nodeType(ctx, child) != NodeType_Node) continue; const name = nodeName(ctx, child); if (std.mem.eql(u8, name, "ImportDecl")) { @@ -380,7 +400,7 @@ fn translateTree(ctx: *TranslationContext) !ast.File { continue; } } - } else if (nodeType(ctx, payload) == C_Parser.LL_NODE_NODE) { + } else if (nodeType(ctx, payload) == NodeType_Node) { const name = nodeName(ctx, payload); if (std.mem.eql(u8, name, "ImportDecl")) { try imports.append(ctx.allocator, try parseImportDecl(ctx, payload)); @@ -402,13 +422,15 @@ fn translateTree(ctx: *TranslationContext) !ast.File { }; } -pub fn translate(allocator: std.mem.Allocator, tree: [*c]C_Parser.ll_tree, file_id: u32) !ast.File { - if (tree == null) return error.ParsingFailed; - const source_ptr: [*]const u8 = @ptrCast(tree.*.input); - const source: []const u8 = source_ptr[0..@intCast(tree.*.input_len)]; +pub fn translate(allocator: std.mem.Allocator, handle: @TypeOf(C_Parser.ParserNew()), file_id: u32) !ast.File { + var source_len: c_int = 0; + const source_ptr = C_Parser.TreeInput(handle, &source_len); + if (source_ptr == null or source_len <= 0) return error.ParsingFailed; + const source_bytes: [*]const u8 = @ptrCast(source_ptr); + const source: []const u8 = source_bytes[0..@intCast(source_len)]; var ctx = TranslationContext{ .allocator = allocator, - .tree = tree, + .handle = handle, .source = source, .file_id = file_id, }; @@ -416,18 +438,16 @@ pub fn translate(allocator: std.mem.Allocator, tree: [*c]C_Parser.ll_tree, file_ } pub fn parseSource(allocator: std.mem.Allocator, file_id: u32, source: []const u8) !ast.File { - const parser = C_Parser.Parser_New(); - defer C_Parser.Parser_Delete(parser); + const handle = C_Parser.ParserNew(); + defer C_Parser.ParserDelete(handle); - var cursor: c_int = 0; - C_Parser.Parser_SetInput(parser, @ptrCast(source.ptr), @intCast(source.len)); - const tree = C_Parser.Parser_Parse(parser, &cursor, null); - if (tree == null) return error.ParsingFailed; - defer C_Parser.ll_tree_free(tree); + if (!C_Parser.ParserParse(handle, @ptrCast(@constCast(source.ptr)), @intCast(source.len))) { + return error.ParsingFailed; + } var ctx = TranslationContext{ .allocator = allocator, - .tree = tree, + .handle = handle, .source = source, .file_id = file_id, }; From 4cc5b1269fc693171a0cc6fff707f91250eb57b5 Mon Sep 17 00:00:00 2001 From: Jefferson Oliveira Date: Thu, 21 May 2026 11:08:11 -0300 Subject: [PATCH 05/11] refactor(parser): remove C parser implementation and header files - Deleted the C parser implementation file `parser.c` and its corresponding header file `parser.h`. - This change reflects the transition to a Go-based parser, aligning with recent updates to the parser architecture and integration with Zig. --- lib/parser.c | 1707 -------------------------------------------------- lib/parser.h | 124 ---- 2 files changed, 1831 deletions(-) delete mode 100644 lib/parser.c delete mode 100644 lib/parser.h diff --git a/lib/parser.c b/lib/parser.c deleted file mode 100644 index 5cc6276..0000000 --- a/lib/parser.c +++ /dev/null @@ -1,1707 +0,0 @@ -/* - * Auto-generated C parser by langlang. - * - * This file embeds the runtime unless RemoveLib=true. - */ - -#include "lib/parser.h" - -/* This tells the embedded runtime sources (c/vm.c, c/tree.c) to not include headers. */ -#define LANGLANG_EMBEDDED 1 - -/* ---- BEGIN embedded runtime: c/vm.c ---- */ -#include -#include -#include -#include -#include - -#ifndef LANGLANG_EMBEDDED -#include "tree.h" -#include "vm.h" -#endif - -void *ll_xrealloc(void *p, size_t n) { - void *q = realloc(p, n); - if (!q && n) { - fprintf(stderr, "vm.c: out of memory\n"); - abort(); - } - return q; -} - -char *ll_xstrdup(const char *s) { - if (!s) return NULL; - size_t n = strlen(s); - char *out = (char *)ll_xrealloc(NULL, n + 1); - memcpy(out, s, n + 1); - return out; -} - -// Returns decoded codepoint in *r and bytes consumed (1..4). On -// invalid input, returns U+FFFD and consumes 1 byte (Go-ish -// behavior). -static int ll_decode_rune(const uint8_t *data, int len, int offset, uint32_t *r) { - if (offset >= len) { - *r = 0; - return 0; - } - uint8_t c0 = data[offset]; - if (c0 < 0x80) { - *r = (uint32_t)c0; - return 1; - } - // 2-byte - if ((c0 & 0xE0) == 0xC0) { - if (offset + 1 >= len) goto invalid; - uint8_t c1 = data[offset + 1]; - if ((c1 & 0xC0) != 0x80) goto invalid; - uint32_t cp = ((uint32_t)(c0 & 0x1F) << 6) | (uint32_t)(c1 & 0x3F); - if (cp < 0x80) goto invalid; // overlong - *r = cp; - return 2; - } - // 3-byte - if ((c0 & 0xF0) == 0xE0) { - if (offset + 2 >= len) goto invalid; - uint8_t c1 = data[offset + 1], c2 = data[offset + 2]; - if ((c1 & 0xC0) != 0x80 || (c2 & 0xC0) != 0x80) goto invalid; - uint32_t cp = ((uint32_t)(c0 & 0x0F) << 12) | - ((uint32_t)(c1 & 0x3F) << 6) | - (uint32_t)(c2 & 0x3F); - if (cp < 0x800) goto invalid; // overlong - if (cp >= 0xD800 && cp <= 0xDFFF) goto invalid; // surrogate - *r = cp; - return 3; - } - // 4-byte - if ((c0 & 0xF8) == 0xF0) { - if (offset + 3 >= len) goto invalid; - uint8_t c1 = data[offset + 1], c2 = data[offset + 2], c3 = data[offset + 3]; - if ((c1 & 0xC0) != 0x80 || (c2 & 0xC0) != 0x80 || (c3 & 0xC0) != 0x80) - goto invalid; - uint32_t cp = ((uint32_t)(c0 & 0x07) << 18) | - ((uint32_t)(c1 & 0x3F) << 12) | - ((uint32_t)(c2 & 0x3F) << 6) | - (uint32_t)(c3 & 0x3F); - if (cp < 0x10000 || cp > 0x10FFFF) goto invalid; // overlong/out of range - *r = cp; - return 4; - } - - invalid: - *r = 0xFFFD; - return 1; -} - -static inline uint16_t ll_decode_u16(const uint8_t *code, int offset) { - return (uint16_t)code[offset] | ((uint16_t)code[offset + 1] << 8); -} - -typedef struct { - uint64_t w[8]; -} ll_bitset512; - -void ll_bitset512_set(ll_bitset512 *b, int id) { - b->w[id >> 6] |= (uint64_t)1u << (id & 63); -} - -bool ll_bitset512_has(const ll_bitset512 *b, int id) { - return (b->w[id >> 6] & ((uint64_t)1u << (id & 63))) != 0; -} - -typedef struct { - uint8_t bits[32]; // 256 bits -} ll_charset; - -bool ll_charset_has_byte(const ll_charset *cs, uint8_t b) { - return (cs->bits[b >> 3] & (uint8_t)(1u << (b & 7))) != 0; -} - -void ll_parsing_error_free(ll_parsing_error *e) { - if (!e) return; - free(e->message); - free(e->label); - memset(e, 0, sizeof(*e)); -} - -enum { LL_EXPECTED_LIMIT = 20 }; - -typedef struct { - uint32_t a, b; -} ll_expected; - -typedef struct { - int cur; - ll_expected arr[LL_EXPECTED_LIMIT]; -} ll_expected_info; - -static void ll_expected_info_clear(ll_expected_info *e) { - e->cur = 0; -} - -static void ll_expected_info_add(ll_expected_info *e, ll_expected s) { - if (e->cur == LL_EXPECTED_LIMIT) return; - if (s.b == 0) { - if (s.a == 0 || s.a == ' ' || s.a == '\n' || s.a == '\r' || s.a == '\t') return; - } - for (int i = 0; i < e->cur; i++) { - if (e->arr[i].a == s.a && e->arr[i].b == s.b) return; - } - e->arr[e->cur++] = s; -} - -typedef struct { int key, val; } ll_i2i; -typedef struct { ll_i2i *items; int len, cap; } ll_i2i_map; - -static void ll_i2i_map_init(ll_i2i_map *m) { memset(m, 0, sizeof(*m)); } -static void ll_i2i_map_free(ll_i2i_map *m) { free(m->items); memset(m, 0, sizeof(*m)); } - -static void ll_i2i_map_put(ll_i2i_map *m, int key, int val) { - for (int i = 0; i < m->len; i++) { - if (m->items[i].key == key) { m->items[i].val = val; return; } - } - if (m->len + 1 > m->cap) { - int cap = m->cap ? m->cap * 2 : 16; - m->items = (ll_i2i *)ll_xrealloc(m->items, (size_t)cap * sizeof(ll_i2i)); - m->cap = cap; - } - m->items[m->len++] = (ll_i2i){.key = key, .val = val}; -} - -static bool ll_i2i_map_get(const ll_i2i_map *m, int key, int *out_val) { - for (int i = 0; i < m->len; i++) { - if (m->items[i].key == key) { *out_val = m->items[i].val; return true; } - } - return false; -} - -typedef struct { char *key; int val; } ll_s2i; -typedef struct { ll_s2i *items; int len, cap; } ll_s2i_map; - -static void ll_s2i_map_init(ll_s2i_map *m) { memset(m, 0, sizeof(*m)); } -static void ll_s2i_map_free(ll_s2i_map *m) { - for (int i = 0; i < m->len; i++) free(m->items[i].key); - free(m->items); - memset(m, 0, sizeof(*m)); -} - -static bool ll_s2i_map_get(const ll_s2i_map *m, const char *key, int *out_val) { - for (int i = 0; i < m->len; i++) { - if (strcmp(m->items[i].key, key) == 0) { *out_val = m->items[i].val; return true; } - } - return false; -} - -static void ll_s2i_map_put(ll_s2i_map *m, const char *key, int val) { - for (int i = 0; i < m->len; i++) { - if (strcmp(m->items[i].key, key) == 0) { m->items[i].val = val; return; } - } - if (m->len + 1 > m->cap) { - int cap = m->cap ? m->cap * 2 : 16; - m->items = (ll_s2i *)ll_xrealloc(m->items, (size_t)cap * sizeof(ll_s2i)); - m->cap = cap; - } - m->items[m->len++] = (ll_s2i){.key = ll_xstrdup(key), .val = val}; -} - -typedef struct { - uint8_t *code; - int code_len; - - const char **strs; - int strs_len; - - ll_charset *sets; - int sets_len; - - ll_expected **sexp; - int *sexp_len; - int sexp_cap; - - ll_s2i_map smap; - ll_i2i_map rxps; - ll_bitset512 rxbs; -} ll_bytecode; - -void ll_bytecode_init(ll_bytecode *b) { - memset(b, 0, sizeof(*b)); - ll_s2i_map_init(&b->smap); - ll_i2i_map_init(&b->rxps); -} - -void ll_bytecode_free(ll_bytecode *b) { - free(b->code); - free((void *)b->strs); - free(b->sets); - if (b->sexp) { - for (int i = 0; i < b->sexp_cap; i++) { - free(b->sexp[i]); - } - } - free(b->sexp); - free(b->sexp_len); - ll_s2i_map_free(&b->smap); - ll_i2i_map_free(&b->rxps); - memset(b, 0, sizeof(*b)); -} - -ll_i2i_map ll_bytecode_compile_error_labels(ll_bytecode *b, const char **labels, const char **messages, int count) { - ll_i2i_map out; - ll_i2i_map_init(&out); - if (count <= 0) return out; - for (int i = 0; i < count; i++) { - int label_id = 0, msg_id = 0; - if (!ll_s2i_map_get(&b->smap, labels[i], &label_id)) continue; - if (!ll_s2i_map_get(&b->smap, messages[i], &msg_id)) { - // append new message to strs (growable copy) - const char **new_strs = (const char **)ll_xrealloc((void *)b->strs, (size_t)(b->strs_len + 1) * sizeof(char *)); - b->strs = new_strs; - b->strs[b->strs_len] = ll_xstrdup(messages[i]); - msg_id = b->strs_len; - b->strs_len++; - ll_s2i_map_put(&b->smap, messages[i], msg_id); - } - ll_i2i_map_put(&out, label_id, msg_id); - } - return out; -} - -// Virtual machine - -typedef enum { - LL_FRAME_BACKTRACKING = 0, - LL_FRAME_CALL = 1, - LL_FRAME_CAPTURE = 2, -} ll_frame_type; - -typedef struct { - int cursor; - uint32_t pc; - uint32_t cap_id; - uint32_t nodes_start; - uint32_t nodes_end; - ll_frame_type t; - bool predicate; -} ll_frame; - -typedef struct { - ll_frame *frames; - int frames_len; - int frames_cap; - - ll_node_id *node_arena; - int node_arena_len; - int node_arena_cap; - - ll_node_id *nodes; - int nodes_len; - int nodes_cap; - - ll_tree *tree; -} ll_stack; - -static void ll_stack_init(ll_stack *s, ll_tree *t) { - memset(s, 0, sizeof(*s)); - s->tree = t; -} - -static void ll_stack_free(ll_stack *s) { - free(s->frames); - free(s->node_arena); - free(s->nodes); - memset(s, 0, sizeof(*s)); -} - -static void ll_stack_reset(ll_stack *s) { - s->frames_len = 0; - s->node_arena_len = 0; - s->nodes_len = 0; -} - -static void ll_stack_grow_frames(ll_stack *s, int need) { - if (s->frames_len + need <= s->frames_cap) return; - int cap = s->frames_cap ? s->frames_cap : 256; - while (cap < s->frames_len + need) cap *= 2; - s->frames = (ll_frame *)ll_xrealloc(s->frames, (size_t)cap * sizeof(ll_frame)); - s->frames_cap = cap; -} - -static void ll_stack_grow_arena(ll_stack *s, int need) { - if (s->node_arena_len + need <= s->node_arena_cap) return; - int cap = s->node_arena_cap ? s->node_arena_cap : 256; - while (cap < s->node_arena_len + need) cap *= 2; - s->node_arena = (ll_node_id *)ll_xrealloc(s->node_arena, (size_t)cap * sizeof(ll_node_id)); - s->node_arena_cap = cap; -} - -static void ll_stack_grow_nodes(ll_stack *s, int need) { - if (s->nodes_len + need <= s->nodes_cap) return; - int cap = s->nodes_cap ? s->nodes_cap : 256; - while (cap < s->nodes_len + need) cap *= 2; - s->nodes = (ll_node_id *)ll_xrealloc(s->nodes, (size_t)cap * sizeof(ll_node_id)); - s->nodes_cap = cap; -} - -static void ll_stack_push(ll_stack *s, ll_frame f) { - ll_stack_grow_frames(s, 1); - f.nodes_start = (uint32_t)s->node_arena_len; - f.nodes_end = f.nodes_start; - s->frames[s->frames_len++] = f; -} - -static ll_frame ll_stack_pop(ll_stack *s) { - ll_frame f = s->frames[s->frames_len - 1]; - s->frames_len--; - return f; -} - -static ll_frame *ll_stack_top(ll_stack *s) { - return &s->frames[s->frames_len - 1]; -} - -static int ll_stack_len(const ll_stack *s) { - return s->frames_len; -} - -static const ll_node_id *ll_stack_frame_nodes(const ll_stack *s, const ll_frame *f, int *out_len) { - *out_len = (int)(f->nodes_end - f->nodes_start); - if (*out_len <= 0) return NULL; - return &s->node_arena[f->nodes_start]; -} - -static void ll_stack_capture(ll_stack *s, const ll_node_id *nodes, int nodes_len) { - if (nodes_len <= 0) return; - int n = s->frames_len; - if (n > 0) { - ll_stack_grow_arena(s, nodes_len); - memcpy(&s->node_arena[s->node_arena_len], nodes, (size_t)nodes_len * sizeof(ll_node_id)); - s->node_arena_len += nodes_len; - s->frames[n - 1].nodes_end = (uint32_t)s->node_arena_len; - return; - } - ll_stack_grow_nodes(s, nodes_len); - memcpy(&s->nodes[s->nodes_len], nodes, (size_t)nodes_len * sizeof(ll_node_id)); - s->nodes_len += nodes_len; -} - -static void ll_stack_capture_one(ll_stack *s, ll_node_id id) { - ll_stack_capture(s, &id, 1); -} - -static void ll_stack_commit_captures_to_parent(ll_stack *s, uint32_t child_start, uint32_t child_end) { - if (child_start == child_end) return; - int n = s->frames_len; - if (n == 0) { - int len = (int)(child_end - child_start); - ll_stack_grow_nodes(s, len); - memcpy(&s->nodes[s->nodes_len], &s->node_arena[child_start], (size_t)len * sizeof(ll_node_id)); - s->nodes_len += len; - } else { - s->frames[n - 1].nodes_end = child_end; - } -} - -static void ll_stack_collect_captures(ll_stack *s) { - int n = s->frames_len; - if (n == 0) return; - ll_frame *f = &s->frames[n - 1]; - if (f->nodes_end > f->nodes_start) { - if (n == 1) { - int len = (int)(f->nodes_end - f->nodes_start); - ll_stack_grow_nodes(s, len); - memcpy(&s->nodes[s->nodes_len], &s->node_arena[f->nodes_start], (size_t)len * sizeof(ll_node_id)); - s->nodes_len += len; - } else { - s->frames[n - 2].nodes_end = f->nodes_end; - } - } -} - -static void ll_stack_truncate_arena(ll_stack *s, uint32_t pos) { - if ((int)pos < 0) pos = 0; - if ((int)pos > s->node_arena_len) pos = (uint32_t)s->node_arena_len; - s->node_arena_len = (int)pos; -} - -typedef struct { - int ffp; - ll_stack stack; - ll_tree tree; - ll_bytecode *bytecode; - bool predicate; - ll_expected_info expected; - bool show_fails; - ll_i2i_map err_labels; - int cap_offset_id; - int cap_offset_start; -} ll_vm; - -typedef enum { - LL_OP_HALT = 0, - LL_OP_ANY, - LL_OP_CHAR, - LL_OP_RANGE, - LL_OP_FAIL, - LL_OP_FAIL_TWICE, - LL_OP_CHOICE, - LL_OP_CHOICE_PRED, - LL_OP_CAP_COMMIT, - LL_OP_CAP_PARTIAL_COMMIT, - LL_OP_CAP_BACK_COMMIT, - LL_OP_CALL, - LL_OP_CAP_RETURN, - LL_OP_JUMP, - LL_OP_THROW, - LL_OP_CAP_BEGIN, - LL_OP_CAP_END, - LL_OP_SET, - LL_OP_SPAN, - LL_OP_CAP_TERM, - LL_OP_CAP_NON_TERM, - LL_OP_COMMIT, - LL_OP_BACK_COMMIT, - LL_OP_PARTIAL_COMMIT, - LL_OP_RETURN, - LL_OP_CAP_TERM_BEGIN_OFFSET, - LL_OP_CAP_NON_TERM_BEGIN_OFFSET, - LL_OP_CAP_END_OFFSET, -} ll_opcode; - -enum { - LL_OP_ANY_SIZE = 1, - LL_OP_CHAR_SIZE = 3, - LL_OP_RANGE_SIZE = 5, - LL_OP_SET_SIZE = 3, - LL_OP_SPAN_SIZE = 3, - LL_OP_CHOICE_SIZE = 3, - LL_OP_COMMIT_SIZE = 3, - LL_OP_FAIL_SIZE = 1, - LL_OP_CALL_SIZE = 4, - LL_OP_RETURN_SIZE = 1, - LL_OP_JUMP_SIZE = 3, - LL_OP_THROW_SIZE = 3, - LL_OP_HALT_SIZE = 1, - LL_OP_CAP_BEGIN_SIZE = 3, - LL_OP_CAP_END_SIZE = 1, - LL_OP_CAP_TERM_SIZE = 3, - LL_OP_CAP_NON_TERM_SIZE = 5, - LL_OP_CAP_TERM_BEGIN_OFFSET_SIZE = 1, - LL_OP_CAP_NON_TERM_BEGIN_OFFSET_SIZE = 3, - LL_OP_CAP_END_OFFSET_SIZE = 1, -}; - -static ll_frame ll_mk_backtrack_frame(int pc, int cursor) { - return (ll_frame){ - .t = LL_FRAME_BACKTRACKING, - .pc = (uint32_t)pc, - .cursor = cursor, - .cap_id = 0, - .predicate = false, - }; -} - -static ll_frame ll_mk_backtrack_pred_frame(int pc, int cursor) { - ll_frame f = ll_mk_backtrack_frame(pc, cursor); - f.predicate = true; - return f; -} - -static ll_frame ll_mk_capture_frame(int id, int cursor) { - return (ll_frame){ - .t = LL_FRAME_CAPTURE, - .cap_id = (uint32_t)id, - .cursor = cursor, - .pc = 0, - .predicate = false, - }; -} - -static ll_frame ll_mk_call_frame(int pc) { - return (ll_frame){.t = LL_FRAME_CALL, .pc = (uint32_t)pc, .cursor = 0, .cap_id = 0, .predicate = false}; -} - -/* we don't want these in tree.h */ -void ll_tree_bind_input(ll_tree *t, const uint8_t *input, int input_len); -void ll_tree_bind_strings(ll_tree *t, const char **strs, int strs_len); -void ll_tree_reset(ll_tree *t); -void ll_tree_set_root(ll_tree *t, ll_node_id id); -ll_node_id ll_tree_add_string(ll_tree *t, int start, int end); -ll_node_id ll_tree_add_sequence(ll_tree *t, const ll_node_id *children, int children_len, int start, int end); -ll_node_id ll_tree_add_node(ll_tree *t, int32_t name_id, ll_node_id child, int start, int end); -ll_node_id ll_tree_add_error(ll_tree *t, int32_t label_id, int32_t message_id, int start, int end); -ll_node_id ll_tree_add_error_with_child(ll_tree *t, int32_t label_id, int32_t message_id, ll_node_id child_id, int start, int end); - -void ll_vm_init(ll_vm *vm, ll_bytecode *bc) { - memset(vm, 0, sizeof(*vm)); - vm->bytecode = bc; - vm->ffp = -1; - ll_tree_init(&vm->tree); - ll_stack_init(&vm->stack, &vm->tree); - ll_i2i_map_init(&vm->err_labels); - ll_tree_bind_strings(&vm->tree, bc->strs, bc->strs_len); -} - -void ll_vm_free(ll_vm *vm) { - ll_stack_free(&vm->stack); - ll_tree_free(&vm->tree); - ll_i2i_map_free(&vm->err_labels); - memset(vm, 0, sizeof(*vm)); -} - -void ll_vm_set_show_fails(ll_vm *vm, bool show_fails) { - vm->show_fails = show_fails; - if (show_fails) { - ll_expected_info_clear(&vm->expected); - } -} - -void ll_vm_set_label_messages(ll_vm *vm, const ll_i2i_map *labels) { - ll_i2i_map_free(&vm->err_labels); - ll_i2i_map_init(&vm->err_labels); - if (!labels) return; - for (int i = 0; i < labels->len; i++) { - ll_i2i_map_put(&vm->err_labels, labels->items[i].key, labels->items[i].val); - } -} - -static void ll_vm_reset(ll_vm *vm) { - ll_stack_reset(&vm->stack); - ll_tree_reset(&vm->tree); - vm->ffp = -1; - if (vm->show_fails) { - ll_expected_info_clear(&vm->expected); - } -} - -static void ll_vm_update_expected(ll_vm *vm, int cursor, ll_expected s) { - bool should_clear = cursor > vm->ffp; - bool should_add = cursor >= vm->ffp; - if (should_clear) ll_expected_info_clear(&vm->expected); - if (should_add) ll_expected_info_add(&vm->expected, s); -} - -static void ll_vm_update_set_expected(ll_vm *vm, int cursor, uint16_t sid) { - bool should_clear = cursor > vm->ffp; - bool should_add = cursor >= vm->ffp; - if (should_clear) ll_expected_info_clear(&vm->expected); - if (!should_add) return; - if (!vm->bytecode->sexp || sid >= (uint16_t)vm->bytecode->sexp_cap) return; - int n = vm->bytecode->sexp_len[sid]; - ll_expected *arr = vm->bytecode->sexp[sid]; - for (int i = 0; i < n && i < LL_EXPECTED_LIMIT; i++) { - ll_expected_info_add(&vm->expected, arr[i]); - } -} - -static void ll_vm_new_term_node(ll_vm *vm, int cursor, int offset) { - if (offset <= 0) return; - int begin = cursor - offset; - ll_node_id nid = ll_tree_add_string(&vm->tree, begin, cursor); - ll_stack_capture_one(&vm->stack, nid); -} - -static void ll_vm_new_non_term_node(ll_vm *vm, int cap_id, int cursor, int offset) { - if (offset <= 0) return; - int begin = cursor - offset; - ll_node_id str_node = ll_tree_add_string(&vm->tree, begin, cursor); - ll_node_id named = ll_tree_add_node(&vm->tree, (int32_t)cap_id, str_node, begin, cursor); - ll_stack_capture_one(&vm->stack, named); -} - -static void ll_vm_new_node(ll_vm *vm, int cursor, ll_frame f, const ll_node_id *nodes, int nodes_len) { - ll_node_id node_id = -1; - bool has_node = false; - bool is_rxp = ll_bitset512_has(&vm->bytecode->rxbs, (int)f.cap_id); - int32_t cap_id = (int32_t)f.cap_id; - int start = f.cursor; - int end = cursor; - - switch (nodes_len) { - case 0: - if (cursor - f.cursor > 0) { - node_id = ll_tree_add_string(&vm->tree, start, end); - has_node = true; - } else if (!is_rxp) { - return; - } - break; - case 1: - node_id = nodes[0]; - has_node = true; - break; - default: - node_id = ll_tree_add_sequence(&vm->tree, nodes, nodes_len, start, end); - has_node = true; - break; - } - - if (is_rxp) { - int msg_id = (int)f.cap_id; - int tmp = 0; - if (ll_i2i_map_get(&vm->err_labels, (int)f.cap_id, &tmp)) msg_id = tmp; - ll_node_id err_node; - if (has_node) { - err_node = ll_tree_add_error_with_child(&vm->tree, cap_id, (int32_t)msg_id, node_id, start, end); - } else { - err_node = ll_tree_add_error(&vm->tree, cap_id, (int32_t)msg_id, start, end); - } - ll_stack_capture_one(&vm->stack, err_node); - return; - } - - if (!has_node) return; - - if (f.cap_id == 0) { - ll_stack_capture_one(&vm->stack, node_id); - return; - } - - ll_node_id named = ll_tree_add_node(&vm->tree, cap_id, node_id, start, end); - ll_stack_capture_one(&vm->stack, named); -} - -static ll_parsing_error ll_vm_mk_err(ll_vm *vm, const uint8_t *data, int data_len, int err_label_id, int cursor, int err_cursor) { - bool is_eof = cursor >= data_len; - uint32_t c = 0; - if (!is_eof) { - (void)ll_decode_rune(data, data_len, cursor, &c); - } - - // Build message (simple dynamic string builder) - size_t cap = 256, len = 0; - char *msg = (char *)ll_xrealloc(NULL, cap); - msg[0] = 0; - - int mapped = 0; - if (ll_i2i_map_get(&vm->err_labels, err_label_id, &mapped) && - mapped >= 0 && mapped < vm->bytecode->strs_len) { - const char *s = vm->bytecode->strs[mapped]; - size_t sl = strlen(s); - if (len + sl + 1 > cap) { - while (len + sl + 1 > cap) cap *= 2; - msg = (char *)ll_xrealloc(msg, cap); - } - memcpy(msg + len, s, sl); - len += sl; - msg[len] = 0; - } else { - if (err_label_id > 0 && err_label_id < vm->bytecode->strs_len) { - const char *lab = vm->bytecode->strs[err_label_id]; - size_t need = strlen(lab) + 4; - if (len + need + 1 > cap) { - while (len + need + 1 > cap) cap *= 2; - msg = (char *)ll_xrealloc(msg, cap); - } - msg[len++] = '['; - memcpy(msg + len, lab, strlen(lab)); - len += strlen(lab); - msg[len++] = ']'; - msg[len++] = ' '; - msg[len] = 0; - } - - if (vm->show_fails && vm->expected.cur > 0) { - const char *pfx = "Expected "; - size_t pl = strlen(pfx); - if (len + pl + 1 > cap) { while (len + pl + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, pfx, pl); len += pl; msg[len] = 0; - - for (int i = 0; i < vm->expected.cur; i++) { - ll_expected e = vm->expected.arr[i]; - char buf[64]; - if (e.b != 0) { - snprintf(buf, sizeof(buf), "'%c-%c'%s", (char)e.a, (char)e.b, (i < vm->expected.cur - 1) ? ", " : ""); - } else { - snprintf(buf, sizeof(buf), "'%c'%s", (char)e.a, (i < vm->expected.cur - 1) ? ", " : ""); - } - size_t bl = strlen(buf); - if (len + bl + 1 > cap) { while (len + bl + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, buf, bl); len += bl; msg[len] = 0; - } - - const char *mid = " but got "; - size_t ml = strlen(mid); - if (len + ml + 1 > cap) { while (len + ml + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, mid, ml); len += ml; msg[len] = 0; - } else { - const char *pfx = "Unexpected "; - size_t pl = strlen(pfx); - if (len + pl + 1 > cap) { while (len + pl + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, pfx, pl); len += pl; msg[len] = 0; - } - - if (is_eof) { - const char *eof = "EOF"; - size_t el = strlen(eof); - if (len + el + 1 > cap) { while (len + el + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, eof, el); len += el; msg[len] = 0; - } else { - char buf[16]; - // printable-ish; mirrors Go's single-rune quoting - if (c <= 0x7F && c != '\'' && c != '\\') { - snprintf(buf, sizeof(buf), "'%c'", (char)c); - } else if (c == '\'') { - snprintf(buf, sizeof(buf), "'\\''"); - } else if (c == '\\') { - snprintf(buf, sizeof(buf), "'\\\\'"); - } else { - snprintf(buf, sizeof(buf), "'?'"); - } - size_t bl = strlen(buf); - if (len + bl + 1 > cap) { while (len + bl + 1 > cap) cap *= 2; msg = (char *)ll_xrealloc(msg, cap); } - memcpy(msg + len, buf, bl); len += bl; msg[len] = 0; - } - } - - const char *lab = ""; - if (err_label_id > 0 && err_label_id < vm->bytecode->strs_len) { - lab = vm->bytecode->strs[err_label_id]; - } - - return (ll_parsing_error){ - .message = msg, - .label = ll_xstrdup(lab), - .start = cursor, - .end = err_cursor, - }; -} - -ll_tree *ll_vm_match_rule(ll_vm *vm, const uint8_t *data, int data_len, int rule_address, int *out_cursor, ll_parsing_error *out_err) { - ll_vm_reset(vm); - ll_tree_bind_input(&vm->tree, data, data_len); - const uint8_t *code = vm->bytecode->code; - int ilen = data_len; - int cursor = 0; - int pc = 0; - - if (rule_address > 0) { - ll_stack_push(&vm->stack, ll_mk_call_frame(pc + LL_OP_CALL_SIZE)); - pc = rule_address; - } - - code_loop: - for (;;) { - uint8_t op = code[pc]; - switch (op) { - case LL_OP_HALT: { - if (vm->stack.nodes_len > 0) { - ll_node_id nid = vm->stack.nodes[vm->stack.nodes_len - 1]; - ll_tree_set_root(&vm->tree, nid); - } - *out_cursor = cursor; - if (out_err) memset(out_err, 0, sizeof(*out_err)); - return &vm->tree; - } - case LL_OP_ANY: { - if (cursor >= ilen) goto fail; - uint32_t r = 0; - int s = ll_decode_rune(data, ilen, cursor, &r); - cursor += s; - pc += LL_OP_ANY_SIZE; - break; - } - case LL_OP_CHAR: { - uint32_t e = (uint32_t)ll_decode_u16(code, pc + 1); - if (cursor >= ilen) goto fail; - uint32_t c = 0; - int s = ll_decode_rune(data, ilen, cursor, &c); - if (c != e) { - if (vm->show_fails) ll_vm_update_expected(vm, cursor, (ll_expected){.a = e, .b = 0}); - goto fail; - } - cursor += s; - pc += LL_OP_CHAR_SIZE; - break; - } - case LL_OP_RANGE: { - if (cursor >= ilen) goto fail; - uint32_t c = 0; - int s = ll_decode_rune(data, ilen, cursor, &c); - uint32_t a = (uint32_t)ll_decode_u16(code, pc + 1); - uint32_t b = (uint32_t)ll_decode_u16(code, pc + 3); - if (c < a || c > b) { - if (vm->show_fails) ll_vm_update_expected(vm, cursor, (ll_expected){.a = a, .b = b}); - goto fail; - } - cursor += s; - pc += LL_OP_RANGE_SIZE; - break; - } - case LL_OP_SET: { - if (cursor >= ilen) goto fail; - uint8_t c = data[cursor]; - uint16_t sid = ll_decode_u16(code, pc + 1); - if (sid >= (uint16_t)vm->bytecode->sets_len || !ll_charset_has_byte(&vm->bytecode->sets[sid], c)) { - if (vm->show_fails) ll_vm_update_set_expected(vm, cursor, sid); - goto fail; - } - cursor++; - pc += LL_OP_SET_SIZE; - break; - } - case LL_OP_SPAN: { - uint16_t sid = ll_decode_u16(code, pc + 1); - if (sid < (uint16_t)vm->bytecode->sets_len) { - ll_charset set = vm->bytecode->sets[sid]; - while (cursor < ilen) { - uint8_t c = data[cursor]; - if (ll_charset_has_byte(&set, c)) { cursor++; continue; } - break; - } - } - pc += LL_OP_SPAN_SIZE; - break; - } - case LL_OP_FAIL: - goto fail; - - case LL_OP_FAIL_TWICE: - (void)ll_stack_pop(&vm->stack); - goto fail; - - case LL_OP_CHOICE: { - int lb = (int)ll_decode_u16(code, pc + 1); - ll_stack_push(&vm->stack, ll_mk_backtrack_frame(lb, cursor)); - pc += LL_OP_CHOICE_SIZE; - break; - } - case LL_OP_CHOICE_PRED: { - int lb = (int)ll_decode_u16(code, pc + 1); - ll_stack_push(&vm->stack, ll_mk_backtrack_pred_frame(lb, cursor)); - pc += LL_OP_CHOICE_SIZE; - vm->predicate = true; - break; - } - case LL_OP_COMMIT: - (void)ll_stack_pop(&vm->stack); - pc = (int)ll_decode_u16(code, pc + 1); - break; - - case LL_OP_BACK_COMMIT: { - ll_frame f = ll_stack_pop(&vm->stack); - cursor = f.cursor; - pc = (int)ll_decode_u16(code, pc + 1); - break; - } - - case LL_OP_PARTIAL_COMMIT: - pc = (int)ll_decode_u16(code, pc + 1); - ll_stack_top(&vm->stack)->cursor = cursor; - break; - - case LL_OP_CALL: - ll_stack_push(&vm->stack, ll_mk_call_frame(pc + LL_OP_CALL_SIZE)); - pc = (int)ll_decode_u16(code, pc + 1); - break; - - case LL_OP_RETURN: { - ll_frame f = ll_stack_pop(&vm->stack); - pc = (int)f.pc; - break; - } - - case LL_OP_JUMP: - pc = (int)ll_decode_u16(code, pc + 1); - break; - - case LL_OP_THROW: { - if (vm->predicate) { - pc += LL_OP_THROW_SIZE; - goto fail; - } - int lb = (int)ll_decode_u16(code, pc + 1); - int addr = 0; - if (ll_i2i_map_get(&vm->bytecode->rxps, lb, &addr)) { - ll_stack_push(&vm->stack, ll_mk_call_frame(pc + LL_OP_THROW_SIZE)); - pc = addr; - continue; - } - *out_cursor = cursor; - if (out_err) *out_err = ll_vm_mk_err(vm, data, data_len, lb, cursor, vm->ffp); - return NULL; - } - - case LL_OP_CAP_BEGIN: { - int id = (int)ll_decode_u16(code, pc + 1); - ll_stack_push(&vm->stack, ll_mk_capture_frame(id, cursor)); - pc += LL_OP_CAP_BEGIN_SIZE; - break; - } - - case LL_OP_CAP_END: { - ll_frame f = ll_stack_pop(&vm->stack); - int nodes_len = 0; - const ll_node_id *nodes = ll_stack_frame_nodes(&vm->stack, &f, &nodes_len); - ll_stack_truncate_arena(&vm->stack, f.nodes_start); - ll_vm_new_node(vm, cursor, f, nodes, nodes_len); - pc += LL_OP_CAP_END_SIZE; - break; - } - - case LL_OP_CAP_TERM: - ll_vm_new_term_node(vm, cursor, (int)ll_decode_u16(code, pc + 1)); - pc += LL_OP_CAP_TERM_SIZE; - break; - - case LL_OP_CAP_NON_TERM: { - int id = (int)ll_decode_u16(code, pc + 1); - int offset = (int)ll_decode_u16(code, pc + 3); - ll_vm_new_non_term_node(vm, id, cursor, offset); - pc += LL_OP_CAP_NON_TERM_SIZE; - break; - } - - case LL_OP_CAP_TERM_BEGIN_OFFSET: - vm->cap_offset_id = -1; - vm->cap_offset_start = cursor; - pc += LL_OP_CAP_TERM_BEGIN_OFFSET_SIZE; - break; - - case LL_OP_CAP_NON_TERM_BEGIN_OFFSET: - vm->cap_offset_id = (int)ll_decode_u16(code, pc + 1); - vm->cap_offset_start = cursor; - pc += LL_OP_CAP_NON_TERM_BEGIN_OFFSET_SIZE; - break; - - case LL_OP_CAP_END_OFFSET: { - int offset = cursor - vm->cap_offset_start; - pc += LL_OP_CAP_END_OFFSET_SIZE; - if (vm->cap_offset_id < 0) { - ll_vm_new_term_node(vm, cursor, offset); - continue; - } - ll_vm_new_non_term_node(vm, vm->cap_offset_id, cursor, offset); - break; - } - - case LL_OP_CAP_COMMIT: { - ll_frame f = ll_stack_pop(&vm->stack); - ll_stack_commit_captures_to_parent(&vm->stack, f.nodes_start, f.nodes_end); - pc = (int)ll_decode_u16(code, pc + 1); - break; - } - - case LL_OP_CAP_BACK_COMMIT: { - ll_frame f = ll_stack_pop(&vm->stack); - ll_stack_commit_captures_to_parent(&vm->stack, f.nodes_start, f.nodes_end); - cursor = f.cursor; - pc = (int)ll_decode_u16(code, pc + 1); - break; - } - - case LL_OP_CAP_PARTIAL_COMMIT: { - pc = (int)ll_decode_u16(code, pc + 1); - ll_frame *top = ll_stack_top(&vm->stack); - top->cursor = cursor; - ll_stack_collect_captures(&vm->stack); - top->nodes_start = (uint32_t)vm->stack.node_arena_len; - top->nodes_end = top->nodes_start; - break; - } - - case LL_OP_CAP_RETURN: { - ll_frame f = ll_stack_pop(&vm->stack); - ll_stack_commit_captures_to_parent(&vm->stack, f.nodes_start, f.nodes_end); - pc = (int)f.pc; - break; - } - - default: - fprintf(stderr, "NO ENTIENDO SENOR: unknown opcode 0x%02x at pc=%d\n", op, pc); - abort(); - } - } - - fail: - if (cursor > vm->ffp) vm->ffp = cursor; - while (ll_stack_len(&vm->stack) > 0) { - ll_frame f = ll_stack_pop(&vm->stack); - ll_stack_truncate_arena(&vm->stack, f.nodes_start); - if (f.t == LL_FRAME_BACKTRACKING) { - pc = (int)f.pc; - vm->predicate = f.predicate; - cursor = f.cursor; - goto code_loop; - } - } - - *out_cursor = cursor; - if (out_err) { - *out_err = ll_vm_mk_err(vm, data, data_len, 0, cursor, vm->ffp); - } - return NULL; -} - -ll_tree *ll_vm_match(ll_vm *vm, const uint8_t *data, int data_len, int *out_cursor, ll_parsing_error *out_err) { - return ll_vm_match_rule(vm, data, data_len, 0, out_cursor, out_err); -} - -/* ---- END embedded runtime: c/vm.c ---- */ - -/* ---- BEGIN embedded runtime: c/tree.c ---- */ -#include -#include -#include - -#ifndef LANGLANG_EMBEDDED -#include "tree.h" -#include "vm.h" -#endif - -void *ll_xrealloc(void *p, size_t n); -char *ll_xstrdup(const char *s); - -void ll_tree_init(ll_tree *t) { - memset(t, 0, sizeof(*t)); - t->root = 0; - t->has_root = false; -} - -void ll_tree_free(ll_tree *t) { - free(t->nodes); - free(t->children); - free(t->child_ranges); - memset(t, 0, sizeof(*t)); -} - -void ll_tree_reset(ll_tree *t) { - t->nodes_len = 0; - t->children_len = 0; - t->child_ranges_len = 0; - t->has_root = false; - t->root = 0; -} - -void ll_tree_set_root(ll_tree *t, ll_node_id id) { - t->root = id; - t->has_root = true; -} - -bool ll_tree_root(const ll_tree *t, ll_node_id *out_id) { - if (!t->has_root || t->nodes_len == 0) return false; - *out_id = t->root; - return true; -} - -ll_node_type ll_tree_type(const ll_tree *t, ll_node_id id) { - return t->nodes[id].typ; -} - -const char *ll_tree_name(const ll_tree *t, ll_node_id id) { - int32_t nid = t->nodes[id].name_id; - if (nid < 0 || nid >= t->strs_len) return ""; - return t->strs[nid]; -} - -ll_range ll_tree_range(const ll_tree *t, ll_node_id id) { - ll_node *n = &t->nodes[id]; - return (ll_range){.start = n->start, .end = n->end}; -} - -bool ll_tree_child(const ll_tree *t, ll_node_id id, ll_node_id *out_child) { - const ll_node *n = &t->nodes[id]; - if (n->child_id < 0) return false; - if (n->typ != LL_NODE_NODE && n->typ != LL_NODE_ERROR) return false; - *out_child = (ll_node_id)n->child_id; - return true; -} - -int ll_tree_children_len(const ll_tree *t, ll_node_id id) { - const ll_node *n = &t->nodes[id]; - if (n->typ == LL_NODE_SEQUENCE) { - if (n->child_id < 0) return 0; - ll_child_range cr = t->child_ranges[n->child_id]; - return (int)(cr.end - cr.start); - } - if (n->typ == LL_NODE_NODE || n->typ == LL_NODE_ERROR) { - return (n->child_id < 0) ? 0 : 1; - } - return 0; -} - -bool ll_tree_children_at(const ll_tree *t, ll_node_id id, int idx, ll_node_id *out_child) { - const ll_node *n = &t->nodes[id]; - if (idx < 0) return false; - if (n->typ == LL_NODE_SEQUENCE) { - if (n->child_id < 0) return false; - ll_child_range cr = t->child_ranges[n->child_id]; - int nchild = (int)(cr.end - cr.start); - if (idx >= nchild) return false; - *out_child = t->children[cr.start + idx]; - return true; - } - if (n->typ == LL_NODE_NODE || n->typ == LL_NODE_ERROR) { - if (n->child_id < 0 || idx != 0) return false; - *out_child = (ll_node_id)n->child_id; - return true; - } - return false; -} - -int ll_tree_children(const ll_tree *t, ll_node_id id, ll_node_id *out, int out_cap) { - int n = ll_tree_children_len(t, id); - if (!out || out_cap <= 0) return n; - int m = n < out_cap ? n : out_cap; - for (int i = 0; i < m; i++) { - ll_node_id child = 0; - if (!ll_tree_children_at(t, id, i, &child)) break; - out[i] = child; - } - return n; -} - -char *ll_tree_text(const ll_tree *t, ll_node_id id) { - const ll_node *n = &t->nodes[id]; - switch (n->typ) { - case LL_NODE_STRING: { - int start = n->start, end = n->end; - if (start < 0) start = 0; - if (end > t->input_len) end = t->input_len; - if (end < start) end = start; - int len = end - start; - char *out = (char *)ll_xrealloc(NULL, (size_t)len + 1); - memcpy(out, t->input + start, (size_t)len); - out[len] = 0; - return out; - } - case LL_NODE_SEQUENCE: { - if (n->child_id < 0) return ll_xstrdup(""); - ll_child_range cr = t->child_ranges[n->child_id]; - // naive O(total) concatenation - size_t cap = 64, cur = 0; - char *buf = (char *)ll_xrealloc(NULL, cap); - buf[0] = 0; - for (int32_t i = cr.start; i < cr.end; i++) { - char *part = ll_tree_text(t, t->children[i]); - size_t plen = strlen(part); - if (cur + plen + 1 > cap) { - while (cur + plen + 1 > cap) cap *= 2; - buf = (char *)ll_xrealloc(buf, cap); - } - memcpy(buf + cur, part, plen); - cur += plen; - buf[cur] = 0; - free(part); - } - return buf; - } - case LL_NODE_NODE: - case LL_NODE_ERROR: { - if (n->child_id < 0) { - if (n->typ == LL_NODE_ERROR) { - // mimic Go: "error[