diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index adbee7d07dec..c5888c656d4d 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -17,6 +17,7 @@ var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); const crippled = switch (builtin.zig_backend) { .stage2_powerpc, .stage2_riscv64, + .stage2_loongarch, => true, else => false, }; @@ -309,7 +310,7 @@ pub fn mainSimple() anyerror!void { stderr.writeAll("... ") catch {}; stderr.writeAll("PASS\n") catch {}; } - } else |err| if (enable_print) { + } else |err| { if (enable_print) { stderr.writeAll(test_fn.name) catch {}; stderr.writeAll("... ") catch {}; diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig index d262a70293b9..bd8368e25de0 100644 --- a/lib/std/builtin.zig +++ b/lib/std/builtin.zig @@ -1125,6 +1125,9 @@ pub const CompilerBackend = enum(u64) { /// The reference implementation self-hosted compiler of Zig, using the /// powerpc backend. stage2_powerpc = 12, + /// The reference implementation self-hosted compiler of Zig, using the + /// loongarch backend. + stage2_loongarch = 13, _, }; diff --git a/lib/std/debug.zig b/lib/std/debug.zig index dbf8e110a28d..e06c9570ac67 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -605,6 +605,7 @@ pub fn defaultPanic( switch (builtin.zig_backend) { .stage2_aarch64, .stage2_arm, + .stage2_loongarch, .stage2_powerpc, .stage2_riscv64, .stage2_spirv, diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 023430d11000..50309214008b 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -2300,6 +2300,125 @@ pub const R_PPC64 = enum(u32) { _, }; +pub const R_LARCH = enum(u32) { + NONE = 0, + @"32" = 1, + @"64" = 2, + RELATIVE = 3, + COPY = 4, + JUMP_SLOT = 5, + TLS_DTPMOD32 = 6, + TLS_DTPMOD64 = 7, + TLS_DTPREL32 = 8, + TLS_DTPREL64 = 9, + TLS_TPREL32 = 10, + TLS_TPREL64 = 11, + IRELATIVE = 12, + TLS_DESC32 = 13, + TLS_DESC64 = 14, + MARK_LA = 20, + MARK_PCREL = 21, + SOP_PUSH_PCREL = 22, + SOP_PUSH_ABSOLUTE = 23, + SOP_PUSH_DUP = 24, + SOP_PUSH_GPREL = 25, + SOP_PUSH_TLS_TPREL = 26, + SOP_PUSH_TLS_GOT = 27, + SOP_PUSH_TLS_GD = 28, + SOP_PUSH_PLT_PCREL = 29, + SOP_ASSERT = 30, + SOP_NOT = 31, + SOP_SUB = 32, + SOP_SL = 33, + SOP_SR = 34, + SOP_ADD = 35, + SOP_AND = 36, + SOP_IF_ELSE = 37, + SOP_POP_32_S_10_5 = 38, + SOP_POP_32_U_10_12 = 39, + SOP_POP_32_S_10_12 = 40, + SOP_POP_32_S_10_16 = 41, + SOP_POP_32_S_10_16_S = 42, + SOP_POP_32_S_5_20 = 43, + SOP_POP_32_S_0_5_10_16_S2 = 44, + SOP_POP_32_S_0_10_10 = 45, + SOP_POP_32_U = 46, + ADD8 = 47, + ADD16 = 48, + ADD24 = 49, + ADD32 = 50, + ADD64 = 51, + SUB8 = 52, + SUB16 = 53, + SUB24 = 54, + SUB32 = 55, + SUB64 = 56, + GNU_VTINHERIT = 57, + GNU_VTENTRY = 58, + B16 = 64, + B21 = 65, + B26 = 66, + ABS_HI20 = 67, + ABS_LO12 = 68, + ABS64_LO20 = 69, + ABS64_HI12 = 70, + PCALA_HI20 = 71, + PCALA_LO12 = 72, + PCALA64_LO20 = 73, + PCALA64_HI12 = 74, + GOT_PC_HI20 = 75, + GOT_PC_LO12 = 76, + GOT64_PC_LO20 = 77, + GOT64_PC_HI12 = 78, + GOT_HI20 = 79, + GOT_LO12 = 80, + GOT64_LO20 = 81, + GOT64_HI12 = 82, + TLS_LE_HI20 = 83, + TLS_LE_LO12 = 84, + TLS_LE64_LO20 = 85, + TLS_LE64_HI12 = 86, + TLS_IE_PC_HI20 = 87, + TLS_IE_PC_LO12 = 88, + TLS_IE64_PC_LO20 = 89, + TLS_IE64_PC_HI12 = 90, + TLS_IE_HI20 = 91, + TLS_IE_LO12 = 92, + TLS_IE64_LO20 = 93, + TLS_IE64_HI12 = 94, + TLS_LD_PC_HI20 = 95, + TLS_LD_HI20 = 96, + TLS_GD_PC_HI20 = 97, + TLS_GD_HI20 = 98, + @"32_PCREL" = 99, + RELAX = 100, + ALIGN = 102, + PCREL20_S2 = 103, + ADD6 = 105, + SUB6 = 106, + ADD_ULEB128 = 107, + SUB_ULEB128 = 108, + @"64_PCREL" = 109, + CALL36 = 110, + TLS_DESC_PC_HI20 = 111, + TLS_DESC_PC_LO12 = 112, + TLS_DESC64_PC_LO20 = 113, + TLS_DESC64_PC_HI12 = 114, + TLS_DESC_HI20 = 115, + TLS_DESC_LO12 = 116, + TLS_DESC64_LO20 = 117, + TLS_DESC64_HI12 = 118, + TLS_DESC_LD = 119, + TLS_DESC_CALL = 120, + TLS_LE_HI20_R = 121, + TLS_LE_ADD_R = 122, + TLS_LE_LO12_R = 123, + TLS_LD_PCREL20_S2 = 124, + TLS_GD_PCREL20_S2 = 125, + TLS_DESC_PCREL20_S2 = 126, + _, +}; + pub const STV = enum(u2) { DEFAULT = 0, INTERNAL = 1, diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 978ae71b74f3..0222156695f5 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -678,6 +678,7 @@ const eqlBytes_allowed = switch (builtin.zig_backend) { // These backends don't support vectors yet. .stage2_powerpc, .stage2_riscv64, + .stage2_loongarch, => false, // The SPIR-V backend does not support the optimized path yet. .stage2_spirv => false, diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index f69fc8f34844..309fdfe8769c 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -510,6 +510,7 @@ const extern_getauxval = switch (builtin.zig_backend) { .stage2_powerpc, .stage2_riscv64, .stage2_sparc64, + .stage2_loongarch, => false, else => !builtin.link_libc, }; diff --git a/lib/std/start.zig b/lib/std/start.zig index a74b83fe8194..fb022338c15e 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -17,6 +17,7 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start"; pub const simplified_logic = switch (builtin.zig_backend) { .stage2_aarch64, .stage2_arm, + .stage2_loongarch, .stage2_powerpc, .stage2_sparc64, .stage2_spirv, @@ -158,6 +159,14 @@ fn exit2(code: usize) noreturn { : "o0", "o1", "o2", "o3", "o4", "o5", "o6", "o7", "memory" ); }, + .loongarch64 => { + asm volatile ("syscall 0" + : + : [number] "{a7}" (93), + [arg1] "{a0}" (code), + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "memory" + ); + }, else => @compileError("TODO"), }, // exits(0) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 1c7675947940..11b000031de4 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -36,6 +36,7 @@ pub const backend_can_print = switch (builtin.zig_backend) { .stage2_powerpc, .stage2_riscv64, .stage2_spirv, + .stage2_loongarch, => false, else => true, }; diff --git a/lib/ubsan_rt.zig b/lib/ubsan_rt.zig index a2e0a6c1aa4c..3c74cd8d6c4e 100644 --- a/lib/ubsan_rt.zig +++ b/lib/ubsan_rt.zig @@ -673,6 +673,7 @@ fn exportHandlerWithAbort( const can_build_ubsan = switch (builtin.zig_backend) { .stage2_powerpc, .stage2_riscv64, + .stage2_loongarch, => false, else => true, }; diff --git a/src/Zcu.zig b/src/Zcu.zig index cb7e8d2a88fb..d01a4ec78721 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -4590,6 +4590,10 @@ pub fn callconvSupported(zcu: *Zcu, cc: std.builtin.CallingConvention) union(enu .spirv_fragment, .spirv_vertex => target.os.tag == .vulkan, else => false, }, + .stage2_loongarch => switch (cc) { + .loongarch64_lp64 => true, + else => false, + }, }; if (!backend_ok) return .{ .bad_backend = backend }; return .ok; diff --git a/src/arch/loongarch/AsmParser.zig b/src/arch/loongarch/AsmParser.zig new file mode 100644 index 000000000000..d97e402aeeb0 --- /dev/null +++ b/src/arch/loongarch/AsmParser.zig @@ -0,0 +1,790 @@ +//! Inline assembly parser for LoongArch. +//! +//! Global MIR refers to the MIR of the whole function. +//! Local MIR refers to the MIR of the current assembly block. +//! All MIR references, such as that in branch pseudo MIRs, are global +//! MIR indexes. + +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.loongarch_asm); +const mem = std.mem; +const ascii = std.ascii; +const Allocator = mem.Allocator; + +const codegen = @import("../../codegen.zig"); +const Zcu = @import("../../Zcu.zig"); + +const Mir = @import("Mir.zig"); +const abi = @import("abi.zig"); +const bits = @import("bits.zig"); +const encoding = @import("encoding.zig"); +const utils = @import("./utils.zig"); +const Register = bits.Register; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; + +const AsmParser = @This(); + +pt: Zcu.PerThread, +target: *const std.Target, +gpa: std.mem.Allocator, +register_manager: *RegisterManager, + +err_msg: ?*Zcu.ErrorMsg = null, +src_loc: Zcu.LazySrcLoc, +/// The piece of assembly that is being parsed. +/// Setter must clear this field when the value is invalid. +asm_loc: ?[]const u8 = null, + +output_len: usize, +input_len: usize, +clobber_len: usize, + +/// All arguments, from outputs to inputs. +args: std.ArrayListUnmanaged(MCArg) = .empty, +/// Named arguments. Mapping from arg name to index. +arg_map: std.StringHashMapUnmanaged(usize) = .empty, +/// Register locks for clobbers. +clobber_locks: std.ArrayListUnmanaged(RegisterLock) = .empty, + +/// Global MIR index of the first local MIR instruction. +mir_offset: Mir.Inst.Index, +/// Generated MIR instructions. +mir_insts: std.ArrayListUnmanaged(Mir.Inst) = .empty, + +/// Labels in this assembly block. +labels: std.StringHashMapUnmanaged(Label) = .empty, + +const InnerError = codegen.CodeGenError || error{ OutOfRegisters, AsmParseFail }; + +pub fn init(parser: *AsmParser) !void { + try parser.args.ensureTotalCapacity(parser.gpa, parser.output_len + parser.input_len); + try parser.arg_map.ensureTotalCapacity(parser.gpa, @intCast(parser.output_len + parser.input_len)); + try parser.clobber_locks.ensureTotalCapacity(parser.gpa, parser.clobber_len); + // pre-allocate space for MIR instructions. + // 32 is a randomly chosen number. + try parser.mir_insts.ensureTotalCapacity(parser.gpa, 32); +} + +pub fn deinit(parser: *AsmParser) void { + parser.args.deinit(parser.gpa); + parser.arg_map.deinit(parser.gpa); + parser.clobber_locks.deinit(parser.gpa); + parser.mir_insts.deinit(parser.gpa); + + var label_it = parser.labels.valueIterator(); + while (label_it.next()) |label| label.pending_relocs.deinit(parser.gpa); + parser.labels.deinit(parser.gpa); + + parser.* = undefined; +} + +/// Finalizes MC generation. +pub fn finalizeCodeGen(parser: *AsmParser) !void { + for (parser.args.items) |*arg| { + if (arg.reg_lock) |lock| { + parser.register_manager.unlockReg(lock); + arg.reg_lock = null; + } + } + for (parser.clobber_locks.items) |lock| { + parser.register_manager.unlockReg(lock); + } + parser.clobber_locks.clearAndFree(parser.gpa); + + var label_it = parser.labels.iterator(); + while (label_it.next()) |entry| { + const label_name = entry.key_ptr.*; + const label = entry.value_ptr; + if (label.pending_relocs.items.len != 0) + return parser.fail("label undefined: '{s}'", .{label_name}); + } +} + +pub const MCValue = union(enum) { + none, + reg: Register, + imm: i32, + + pub fn format( + mcv: MCValue, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (mcv) { + .none => try writer.print("(none)", .{}), + .reg => |reg| try writer.print("{s}", .{@tagName(reg)}), + .imm => |imm| try writer.print("imm:{}", .{imm}), + } + } +}; + +pub const MCArg = struct { + value: MCValue, + /// Register lock. + /// For non-register values, this is always null. + /// For outputs, null means the output does not require early-clobber and can be reused. + /// For inputs, null means that it reuses an output register. + reg_lock: ?RegisterManager.RegisterLock, + + pub fn format( + mca: MCArg, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + try writer.print("{}", .{mca.value}); + if (mca.reg_lock != null) + try writer.writeAll(" (locked)"); + } +}; + +fn fail(parser: *AsmParser, comptime format: []const u8, args: anytype) error{ OutOfMemory, AsmParseFail } { + @branchHint(.cold); + assert(parser.err_msg == null); + parser.err_msg = try Zcu.ErrorMsg.create(parser.gpa, parser.src_loc, format, args); + if (parser.asm_loc) |asm_loc| { + const asm_str = mem.trim(u8, asm_loc, " \t\n"); + try parser.pt.zcu.errNote(parser.src_loc, parser.err_msg.?, "at '{s}'", .{asm_str}); + } + return error.AsmParseFail; +} + +inline fn hasFeature(parser: *AsmParser, feature: std.Target.loongarch.Feature) bool { + return std.Target.loongarch.featureSetHas(parser.target.cpu.features, feature); +} + +fn parseRegister(parser: *AsmParser, orig_name: []const u8) !?Register { + if (orig_name.len < 2) return null; + if (orig_name[0] == '$') return parseRegister(parser, orig_name[1..]); + + const name = try ascii.allocLowerString(parser.gpa, orig_name); + defer parser.gpa.free(name); + + if (mem.eql(u8, name, "zero")) return .r0; + if (mem.eql(u8, name, "ra")) return .r1; + if (mem.eql(u8, name, "tp")) return .r2; + if (mem.eql(u8, name, "sp")) return .r3; + if (mem.eql(u8, name, "ert")) return .r21; + if (mem.eql(u8, name, "fp")) return .r22; + if (mem.eql(u8, name, "s9")) return .r22; + + if (name[0] == 'v' and !parser.hasFeature(.lsx)) return null; + if (name[0] == 'x' and !parser.hasFeature(.lasx)) return null; + + const min_reg: Register, const max_reg: Register, const num_str: []const u8 = switch (name[0]) { + 'r' => .{ .r0, .r31, name[1..] }, + 'f' => if (mem.startsWith(u8, name, "fcc")) + .{ .fcc0, .fcc7, name[3..] } + else if (mem.startsWith(u8, name, "fa")) + .{ .f0, .f7, name[2..] } + else if (mem.startsWith(u8, name, "ft")) + .{ .f8, .f23, name[2..] } + else if (mem.startsWith(u8, name, "fs")) + .{ .f24, .f31, name[2..] } + else + .{ .f0, .f31, name[1..] }, + 'v' => .{ .v0, .v31, name[1..] }, + 'x' => .{ .x0, .x31, name[1..] }, + 'a' => .{ .r4, .r11, name[1..] }, + 't' => .{ .r12, .r20, name[1..] }, + 's' => .{ .r23, .r31, name[1..] }, + else => return null, + }; + const num = std.fmt.parseUnsigned(usize, num_str, 10) catch return null; + const count: usize = @intCast(@intFromEnum(max_reg) - @intFromEnum(min_reg) + 1); + if (num >= count) return null; + return @enumFromInt(@intFromEnum(min_reg) + num); +} + +/// Parses a constraint code. +/// This allocates needed registers and spills them, but does not lock them. +/// When is_input is set, 'r' constraints will prefer non-locked outputs. +fn parseConstraintCode(parser: *AsmParser, constraint: []u8, is_input: bool) !MCValue { + // known registers + if (constraint[0] == '{') { + if (mem.indexOfScalar(u8, constraint, '}')) |end| { + if (end != constraint.len - 1) + return parser.fail("alternatives in assembly constraint code are not supported in Zig", .{}); + } else { + return parser.fail("invalid register constraint: '{s}'", .{constraint}); + } + const reg_name = constraint["{".len .. constraint.len - "}".len]; + if (try parser.parseRegister(reg_name)) |reg| { + return .{ .reg = reg }; + } else { + return parser.fail("invalid register: '{s}'", .{reg_name}); + } + } + // single-letter constraints + if (constraint[0] == '^') + return parser.fail("'^XX' assembly constraint code is a LLVM dialect and is not supported in Zig", .{}); + if (constraint.len != 1) + return parser.fail("alternatives in assembly constraint code are not supported in Zig", .{}); + switch (constraint[0]) { + 'r' => { + if (is_input) { + for (parser.args.items[0..parser.output_len]) |*arg| { + switch (arg.value) { + .reg => |reg| { + if (arg.reg_lock == null) { + arg.reg_lock = parser.register_manager.lockReg(reg) orelse unreachable; + } + }, + else => {}, + } + } + } + const reg = try parser.register_manager.allocReg(null, abi.getAllocatableRegSet(.int)); + return .{ .reg = reg }; + }, + 'm' => return parser.fail("memory operand in assembly is not allowed in LoongArch", .{}), + 'i', 'n', 's' => return .{ .imm = 0 }, + 'X' => return parser.fail("'X' constraints in assembly are not supported", .{}), + else => return parser.fail("invalid constraint: '{s}'", .{constraint}), + } +} + +/// Parses a output constraint. +/// Locks the register if early clobber is required. +pub fn parseOutputConstraint(parser: *AsmParser, name: []u8, constraint: []u8) !void { + if (constraint[0] == '=') return parser.parseOutputConstraint(name, constraint[1..]); + parser.asm_loc = constraint; + defer parser.asm_loc = null; + + const is_early_clobber = constraint[1] == '&'; + const constraint_code = constraint[@intFromBool(is_early_clobber)..]; + const mcv = try parser.parseConstraintCode(constraint_code, false); + + const reg_lock: ?RegisterManager.RegisterLock = lock: { + if (is_early_clobber) { + switch (mcv) { + .reg => |reg| break :lock parser.register_manager.lockRegAssumeUnused(reg), + else => {}, + } + } + break :lock null; + }; + + switch (mcv) { + .none => unreachable, + .imm => return parser.fail("immediate constraint cannot be used in output: '{s}'", .{constraint}), + else => {}, + } + + if (!mem.eql(u8, name, "_")) + parser.arg_map.putAssumeCapacityNoClobber(name, @intCast(parser.args.items.len)); + parser.args.appendAssumeCapacity(.{ + .value = mcv, + .reg_lock = reg_lock, + }); +} + +/// Parses a input constraint. +/// Locks the register immediately. +pub fn parseInputConstraint(parser: *AsmParser, name: []u8, constraint: []u8) !void { + parser.asm_loc = constraint; + defer parser.asm_loc = null; + + const mcv = mcv: { + // output alias + if (std.fmt.parseUnsigned(usize, constraint, 10)) |arg_i| { + if (arg_i >= parser.args.items.len) + return parser.fail("invalid output alias intput constraint: '{s}'", .{constraint}); + break :mcv parser.args.items[arg_i].value; + } + // constraint code + break :mcv try parser.parseConstraintCode(constraint, true); + }; + + const reg_lock: ?RegisterManager.RegisterLock = switch (mcv) { + .reg => |reg| parser.register_manager.lockReg(reg), + else => null, + }; + + if (!mem.eql(u8, name, "_")) + parser.arg_map.putAssumeCapacityNoClobber(name, @intCast(parser.args.items.len)); + parser.args.appendAssumeCapacity(.{ + .value = mcv, + .reg_lock = reg_lock, + }); +} + +/// Parses a clobber constraint and locks the register. +pub fn parseClobberConstraint(parser: *AsmParser, clobber: []u8) !void { + parser.asm_loc = clobber; + defer parser.asm_loc = null; + + if (std.mem.eql(u8, clobber, "")) return; + if (std.mem.eql(u8, clobber, "cc")) return; + if (std.mem.eql(u8, clobber, "memory")) return; + if (std.mem.eql(u8, clobber, "redzone")) return; + if (try parser.parseRegister(clobber)) |reg| { + try parser.register_manager.getReg(reg, null); + if (parser.register_manager.lockReg(reg)) |lock| { + parser.clobber_locks.appendAssumeCapacity(lock); + } + return; + } + + return parser.fail("invalid clobber: '{s}'", .{clobber}); +} + +/// Finalizes the constraint parsing phase. +/// This locks all unlocked regs in outputs and inputs. +pub fn finalizeConstraints(parser: *AsmParser) !void { + log.debug("{}", .{parser.fmtArgsAndClobbers()}); + // lock remaining output register + // input registers are locked immediately after being parsed. + for (parser.args.items[0..parser.output_len]) |*arg| { + switch (arg.value) { + .reg => |reg| { + if (arg.reg_lock == null) { + arg.reg_lock = parser.register_manager.lockReg(reg) orelse unreachable; + } + }, + else => {}, + } + } +} + +const FormatArgsData = struct { + self: *AsmParser, +}; +fn formatArgs( + data: FormatArgsData, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) @TypeOf(writer).Error!void { + const parser = data.self; + try writer.writeAll("Args:"); + for (parser.args.items, 0..) |*arg, arg_i| { + try writer.print("\n {} = {}", .{ arg_i, arg }); + if (arg.reg_lock != null) try writer.writeAll(" (locked)"); + } + var it = parser.arg_map.iterator(); + while (it.next()) |entry| try writer.print("\n {s} = {}", .{ entry.key_ptr.*, entry.value_ptr.* }); + + try writer.writeAll("\nClobber locks:"); + for (parser.clobber_locks.items) |*arg| { + try writer.print(" {s}", .{@tagName(RegisterManager.regAtTrackedIndex(arg.tracked_index))}); + } +} +fn fmtArgsAndClobbers(self: *AsmParser) std.fmt.Formatter(formatArgs) { + return .{ .data = .{ .self = self } }; +} + +const Label = struct { + /// Local MIR index of the label. + /// For anonymous labels, it refers to the backwards label. + target: ?Mir.Inst.Index = null, + /// Pending relocations, local MIR index. + pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + + const Kind = enum { definition, reference }; + + fn isValid(kind: Kind, name: []const u8) bool { + if (name.len == 0) return false; + if (kind == .reference and Label.isAnonymousRef(name)) return true; + if (kind == .definition and ascii.isDigit(name[0])) return Label.isAnonymousDef(name); + for (name, 0..) |c, i| switch (c) { + else => return false, + '$' => if (i == 0) return false, + '.', '0'...'9', '@', 'A'...'Z', '_', 'a'...'z' => {}, + }; + return true; + } + + fn isAnonymousDef(name: []const u8) bool { + if (name.len == 0) return false; + for (name) |ch| + if (!ascii.isDigit(ch)) return false; + return true; + } + + fn isAnonymousRef(name: []const u8) bool { + if (name.len == 0) return false; + for (name[0 .. name.len - 1]) |ch| + if (!ascii.isDigit(ch)) return false; + return switch (name[name.len - 1]) { + else => false, + 'B', 'F', 'b', 'f' => true, + }; + } +}; + +fn getLabel(parser: *AsmParser, label_name: []const u8) !*Label { + const label_gop = try parser.labels.getOrPut(parser.gpa, label_name); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; + return label_gop.value_ptr; +} + +pub fn parseSource(parser: *AsmParser, source: []const u8) !void { + var line_it = mem.tokenizeAny(u8, source, "\n\r;"); + while (line_it.next()) |line| try parser.parseLine(line); +} + +pub fn parseLine(parser: *AsmParser, line: []const u8) !void { + parser.asm_loc = line; + defer parser.asm_loc = null; + + var line_it = mem.tokenizeAny(u8, line, " \t"); + const mnemonic: []const u8 = while (line_it.next()) |mnemonic_str| { + if (mem.startsWith(u8, mnemonic_str, "#")) return; + if (mem.startsWith(u8, mnemonic_str, "//")) return; + if (mem.endsWith(u8, mnemonic_str, ":")) { + try parser.parseLabel(mnemonic_str[0 .. mnemonic_str.len - ":".len]); + continue; + } + break mnemonic_str; + } else return; + var op_it = OperandIterator.init(parser, line_it.rest()); + const inst = try parser.parseInst(mnemonic, &op_it); + + if (!op_it.isEnd()) + return parser.fail("excessive operand: '{s}'", .{op_it.iter.next().?}); + + log.debug(" | {}: {}", .{ parser.mir_insts.items.len, inst }); + try parser.mir_insts.append(parser.gpa, inst); +} + +fn parseLabel(parser: *AsmParser, label_name: []const u8) !void { + if (!Label.isValid(.definition, label_name)) + return parser.fail("invalid label: '{s}'", .{label_name}); + + const anon = Label.isAnonymousDef(label_name); + if (anon) + log.debug(" | label {s} (anonymous):", .{label_name}) + else + log.debug(" | label {s}:", .{label_name}); + + const label_gop = try parser.labels.getOrPut(parser.gpa, label_name); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else { + const label = label_gop.value_ptr; + + if (!anon and label.target != null) + return parser.fail("redefined label: '{s}'", .{label_name}); + + for (label.pending_relocs.items) |pending_reloc| + parser.performReloc(pending_reloc); + + if (anon) + label.pending_relocs.clearRetainingCapacity() + else + label.pending_relocs.clearAndFree(parser.gpa); + } + label_gop.value_ptr.target = @intCast(parser.mir_insts.items.len); +} + +fn performReloc(parser: *AsmParser, reloc: Mir.Inst.Index) void { + log.debug(" | <-- reloc {}", .{reloc}); + + const next_inst: u32 = @intCast(parser.mir_insts.items.len); + const inst = &parser.mir_insts.items[reloc]; + switch (inst.tag.unwrap()) { + .inst => unreachable, + .pseudo => |tag| { + switch (tag) { + .branch => inst.data.br.inst = parser.mir_offset + next_inst, + else => unreachable, + } + }, + } +} + +const OperandIterator = struct { + parser: *AsmParser, + iter: mem.SplitIterator(u8, .any), + + fn init(parser: *AsmParser, ops: []const u8) OperandIterator { + return .{ + .parser = parser, + .iter = mem.splitAny(u8, ops, ",("), + }; + } + + fn isEnd(it: *OperandIterator) bool { + return it.iter.peek() == null; + } + + fn next(it: *OperandIterator) ?[]const u8 { + if (it.iter.next()) |op| { + return std.mem.trim(u8, op, " \t"); + } else { + return null; + } + } + + fn resolveArg(it: *OperandIterator, tmpl: []const u8) !?*MCValue { + if (tmpl.len < 2) + return null; + if (tmpl[0] == '%') { + if (tmpl[1] == '[' and tmpl[tmpl.len - 1] == ']') { + const arg_name = tmpl[2..][0 .. tmpl.len - 3]; + if (it.parser.arg_map.get(arg_name)) |arg_i| { + return &it.parser.args.items[arg_i].value; + } else { + return it.parser.fail("undefined named assembly argument: '{s}'", .{tmpl}); + } + } else if (std.fmt.parseInt(usize, tmpl[1..], 10) catch null) |arg_i| { + if (arg_i < it.parser.args.items.len) { + return &it.parser.args.items[arg_i].value; + } else { + return it.parser.fail("undefined assembly argument: '{s}'", .{tmpl}); + } + } + } + return null; + } + + fn nextReg(it: *OperandIterator) !Register { + if (it.next()) |name| { + if (try it.resolveArg(name)) |mcv| { + switch (mcv.*) { + .reg => |reg| return reg, + else => return it.parser.fail("argument is not a register: '{s}'", .{name}), + } + } else if (try it.parser.parseRegister(name)) |reg| { + return reg; + } else { + return it.parser.fail("invalid register operand: '{s}'", .{name}); + } + } else return it.parser.fail("missing register operand", .{}); + } + + fn tryNextReg(it: *OperandIterator) !?Register { + return if (it.next()) |name| { + if (try it.resolveArg(name)) |mcv| { + switch (mcv.*) { + .reg => |reg| reg, + else => null, + } + } else try it.parser.parseRegister(name); + } else null; + } + + fn nextImm(it: *OperandIterator, T: type) !T { + if (it.next()) |imm_str| { + if (try it.resolveArg(imm_str)) |mcv| { + switch (mcv.*) { + .imm => |imm| { + if (std.math.cast(T, imm)) |imm_cast| { + return imm_cast; + } else { + return it.parser.fail("immediate argument cannot fit into " ++ @typeName(T) ++ ": '{s}'", .{imm_str}); + } + }, + else => return it.parser.fail("argument is not an immediate: '{s}'", .{imm_str}), + } + } else return std.fmt.parseInt(T, imm_str, 0) catch |err| switch (err) { + error.Overflow => return it.parser.fail("immediate operand cannot fit into " ++ @typeName(T) ++ ": '{s}'", .{imm_str}), + error.InvalidCharacter => return it.parser.fail("invalid " ++ @typeName(T) ++ " operand: '{s}'", .{imm_str}), + }; + } else return it.parser.fail("missing " ++ @typeName(T) ++ " operand", .{}); + } + + fn tryNextImm(it: *OperandIterator, T: type) ?T { + if (it.next()) |imm_str| { + if (try it.resolveArg(imm_str)) |mcv| { + return switch (mcv.*) { + .imm => |imm| { + if (std.math.cast(T, imm)) |imm_cast| + imm_cast + else + null; + }, + else => null, + }; + } else return std.fmt.parseInt(T, imm_str, 0) catch null; + } else return null; + } +}; + +fn parseInst(parser: *AsmParser, mnemonic: []const u8, ops: *OperandIterator) !Mir.Inst { + @setEvalBranchQuota(3_000); + // find override matchers + inline for (@typeInfo(instToMatcher).@"struct".decls) |decl| { + if (mnemonicEql(decl.name, mnemonic)) { + const matcher = @field(instToMatcher, decl.name); + switch (@typeInfo(@TypeOf(matcher))) { + .@"fn" => return matcher(decl.name, ops), + .enum_literal => return InstMatcher.default(ops, @tagName(matcher)), + else => unreachable, + } + } + } + + // find default matchers + inline for (@typeInfo(encoding.Inst).@"struct".decls) |decl| { + // check blocklist + if (@hasField(defaultMatcherBlocklist, decl.name)) continue; + // check name + if (mnemonicEql(decl.name, mnemonic)) + return InstMatcher.default(ops, decl.name); + } + + return parser.fail("invalid mnemonic: '{s}'", .{mnemonic}); +} + +fn mnemonicEql(mnemonic: []const u8, src: []const u8) bool { + if (mnemonic.len != src.len) return false; + for (mnemonic, 0..) |ch, i| { + const src_ch = src[i]; + if (ch == '_' and (src_ch == '_' or src_ch == '.')) continue; + if (ch != ascii.toLower(src_ch)) return false; + } + return true; +} + +const InstMatcher = struct { + fn default(op_iter: *OperandIterator, comptime decl_name: []const u8) !Mir.Inst { + const gen_fn = @field(encoding.Inst, decl_name); + + const ops_ty = std.meta.ArgsTuple(@TypeOf(gen_fn)); + var ops: ops_ty = undefined; + inline for (std.meta.fields(ops_ty)) |op| { + const op_ty = op.type; + if (op_ty == Register) { + @field(ops, op.name) = try op_iter.nextReg(); + } else { + const op_ty_info = @typeInfo(op_ty); + switch (op_ty_info) { + .int => { + @field(ops, op.name) = try op_iter.nextImm(op_ty); + }, + else => unreachable, + } + } + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + const inst: encoding.Inst = @call(.auto, gen_fn, ops); + return .initInst(inst); + } + + pub fn bstr_w(decl_name: []const u8, ops: *OperandIterator) !Mir.Inst { + const op: encoding.OpCode = if (mem.eql(u8, decl_name, "bstrins_w")) .bstrins_w else .bstrpick_w; + const rd = try ops.nextReg(); + const rj = try ops.nextReg(); + const msbw = try ops.nextImm(u5); + const lsbw = try ops.nextImm(u5); + return .initInst(.{ + .opcode = op, + .data = .{ .DJUk5Um5 = .{ rd, rj, lsbw, msbw } }, + }); + } + + pub fn bstr_d(decl_name: []const u8, ops: *OperandIterator) !Mir.Inst { + const op: encoding.OpCode = if (mem.eql(u8, decl_name, "bstrins_d")) .bstrins_d else .bstrpick_d; + const rd = try ops.nextReg(); + const rj = try ops.nextReg(); + const msbw = try ops.nextImm(u6); + const lsbw = try ops.nextImm(u6); + return .initInst(.{ + .opcode = op, + .data = .{ .DJUk6Um6 = .{ rd, rj, lsbw, msbw } }, + }); + } + + pub fn csrxchg(_: []const u8, ops: *OperandIterator) !Mir.Inst { + const rd = try ops.nextReg(); + const rj = try ops.nextReg(); + const csr_num = try ops.nextImm(u14); + if (rj == .r0 or rj == .r1) + return ops.parser.fail("r0 and r1 cannot be used as rj for CSRXCHG", .{}); + return .initInst(.csrxchg(rd, rj, csr_num)); + } + + pub fn cacop(_: []const u8, ops: *OperandIterator) !Mir.Inst { + const code = try ops.nextImm(u5); + const rj = try ops.nextReg(); + const si12 = try ops.nextImm(i12); + return .initInst(.cacop(rj, code, si12)); + } + + pub fn invtlb(_: []const u8, ops: *OperandIterator) !Mir.Inst { + const op = try ops.nextImm(u5); + const rj = try ops.nextReg(); + const rk = try ops.nextReg(); + return .initInst(.tlbinv(rj, rk, op)); + } + + pub fn preld(decl_name: []const u8, ops: *OperandIterator) !Mir.Inst { + const op: encoding.OpCode = if (mem.eql(u8, decl_name, "preld")) .preld else .preldx; + const hint = try ops.nextImm(u5); + const rj = try ops.nextReg(); + const si12 = try ops.nextImm(i12); + return .initInst(.{ + .opcode = op, + .data = .{ .JUd5Sk12 = .{ rj, hint, si12 } }, + }); + } +}; + +/// Maps mnemonics to custom matchers. +const instToMatcher = struct { + pub const bstrins_w = InstMatcher.bstr_w; + pub const bstrpick_w = InstMatcher.bstr_w; + pub const bstrins_d = InstMatcher.bstr_d; + pub const bstrpick_d = InstMatcher.bstr_d; + pub const csrxchg = InstMatcher.csrxchg; + pub const cacop = InstMatcher.cacop; + pub const invtlb = InstMatcher.invtlb; + pub const tlbinv = InstMatcher.invtlb; + pub const preld = InstMatcher.preld; + pub const preldx = InstMatcher.preld; + // pub const b = InstMatcher.branch; + // pub const bl = InstMatcher.branch; + // pub const beqz = InstMatcher.cond_br_zero; + // pub const bnez = InstMatcher.cond_br_zero; + // pub const bceqz = InstMatcher.fcc_cond_br; + // pub const bcnez = InstMatcher.fcc_cond_br; + + pub const dbcl = .dbgcall; + pub const ertn = .eret; + pub const pcaddi = .pcaddu2i; + + pub const ext_w_b = .sext_b; + pub const ext_w_h = .sext_h; + + pub const ldptr_w = .ldox4_w; + pub const ldptr_d = .ldox4_d; + + pub const stptr_w = .stox4_w; + pub const stptr_d = .stox4_d; + + pub const bitrev_w = .revbit_w; + pub const bitrev_d = .revbit_d; + pub const bitrev_4b = .revbit_4b; + pub const bitrev_8b = .revbit_8b; + + pub const asrtle_d = .asrtle; + pub const asrtgt_d = .asrtgt; + + pub const lu32i_d = .cu32i_d; + pub const lu52i = .cu52i_d; + + pub const alsl_w = .sladd_w; + pub const alsl_wu = .sladd_wu; + pub const alsl_d = .sladd_d; + + pub const bytepick_w = .catpick_w; + pub const bytepick_d = .catpick_d; +}; + +/// Mnemonics that will not have auto-generated matcher. +const defaultMatcherBlocklist = enum { + b, + bl, + beqz, + bnez, + bceqz, + bcnez, + bgt, + bgtu, + ble, + bleu, +}; diff --git a/src/arch/loongarch/CodeGen.zig b/src/arch/loongarch/CodeGen.zig new file mode 100644 index 000000000000..e1953cf4ecf0 --- /dev/null +++ b/src/arch/loongarch/CodeGen.zig @@ -0,0 +1,4183 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const cast = std.math.cast; + +const Air = @import("../../Air.zig"); +const codegen = @import("../../codegen.zig"); +const InternPool = @import("../../InternPool.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); +const Module = @import("../../Package/Module.zig"); +const Type = @import("../../Type.zig"); +const Value = @import("../../Value.zig"); + +const abi = @import("./abi.zig"); +const bits = @import("./bits.zig"); +const Mir = @import("./Mir.zig"); +const Lir = @import("./Lir.zig"); +const Emit = @import("./Emit.zig"); +const encoding = @import("./encoding.zig"); +const AsmParser = @import("./AsmParser.zig"); +const RegisterManager = abi.RegisterManager; +const Register = bits.Register; +const FrameIndex = bits.FrameIndex; + +const assert = std.debug.assert; +const log = std.log.scoped(.codegen); +const cg_mir_log = std.log.scoped(.codegen_mir); +const cg_select_log = std.log.scoped(.codegen_select); +const tracking_log = std.log.scoped(.tracking); +const verbose_tracking_log = std.log.scoped(.verbose_tracking); + +const CodeGen = @This(); + +const InnerError = codegen.CodeGenError || error{OutOfRegisters}; + +const err_ret_trace_index: Air.Inst.Index = @enumFromInt(std.math.maxInt(u32)); + +gpa: Allocator, +pt: Zcu.PerThread, +air: Air, +liveness: Air.Liveness, + +target: *const std.Target, +owner: Owner, +inline_func: InternPool.Index, +mod: *Module, +fn_type: Type, + +// Call infos +call_info: abi.CallInfo, +arg_index: u32, +// MCVs of arguments. +// For ref_frame CCVs, the MCV is the pointer in load_frame. +args_mcv: []MCValue, +ret_mcv: MCValue, + +src_loc: Zcu.LazySrcLoc, + +/// MIR instructions +mir_instructions: std.MultiArrayList(Mir.Inst) = .empty, +epilogue_label: Mir.Inst.Index = undefined, + +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, +inst_tracking: InstTrackingMap = .empty, +register_manager: RegisterManager = .{}, + +// Key is the block instruction +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockState) = .empty, + +/// Generation of the current scope, increments by 1 for every entered scope. +scope_generation: u32 = 0, + +frame_allocs: std.MultiArrayList(FrameAlloc) = .empty, +free_frame_indices: std.ArrayListUnmanaged(FrameIndex) = .empty, +frame_locs: std.MultiArrayList(Mir.FrameLoc) = .empty, + +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { + /// The state to restore before branching. + state: State, + /// The branch target. + target: Mir.Inst.Index, +}) = .empty, + +next_temp_index: Temp.Index = @enumFromInt(0), +temp_type: [Temp.Index.max]Type = undefined, + +const Owner = union(enum) { + nav_index: InternPool.Nav.Index, + lazy_sym: link.File.LazySymbol, +}; + +pub const MCValue = union(enum) { + /// No runtime bits / not available yet. + none, + /// Unreachable. + /// CFG will not allow this value to be observed. + unreach, + /// No more references to this value remain. + /// The payload is the value of scope_generation at the point where the death occurred + dead: u32, + /// The value is undefined. + undef, + /// A pointer-sized integer that fits in a register. + /// If the type is a pointer, this is the pointer address in virtual address space. + immediate: u64, + /// The value is in a register. + register: Register, + /// The value is split across two registers. + register_pair: [2]Register, + /// The value is split across three registers. + register_triple: [3]Register, + /// The value is split across four registers. + register_quadruple: [4]Register, + /// The value is a constant offset plus the value in a register. + register_bias: bits.RegisterOffset, + /// The value is in memory at a hard-coded address. + /// If the type is a pointer, it means the pointer address is at this memory location. + memory: u64, + /// The value stored at an offset from a frame index + /// Payload is a frame address. + load_frame: bits.FrameAddr, + /// The address of an offset from a frame index + /// Payload is a frame address. + lea_frame: bits.FrameAddr, + /// A value whose lower-ordered bits are in a register and the others are in a frame. + register_frame: bits.RegisterFrame, + /// The value is in memory at a constant offset from the address in a register. + register_offset: bits.RegisterOffset, + /// The value is stored as a NAV. + load_nav: bits.NavOffset, + /// The value is the address of a NAV. + lea_nav: bits.NavOffset, + /// The value is stored as a UAV. + load_uav: bits.UavOffset, + /// The value is the address of a UAV. + lea_uav: bits.UavOffset, + /// The value is a lazy symbol. + load_lazy_sym: link.File.LazySymbol, + /// The value is the address of a lazy symbol. + lea_lazy_sym: link.File.LazySymbol, + /// This indicates that we have already allocated a frame index for this instruction, + /// but it has not been spilled there yet in the current control flow. + reserved_frame: FrameIndex, + /// Reference to the value of another AIR. + air_ref: Air.Inst.Ref, + + fn isModifiable(mcv: MCValue) bool { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register_bias, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .reserved_frame, + .air_ref, + => false, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .memory, + .register_frame, + .register_offset, + => true, + .load_frame => |frame_addr| !frame_addr.index.isNamed(), + }; + } + + fn isMemory(mcv: MCValue) bool { + return switch (mcv) { + .memory, .register_offset, .load_frame, .load_nav, .load_uav, .load_lazy_sym => true, + else => false, + }; + } + + fn isImmediate(mcv: MCValue) bool { + return switch (mcv) { + .immediate => true, + else => false, + }; + } + + fn isRegister(mcv: MCValue) bool { + return switch (mcv) { + .register => true, + .register_bias => |reg_off| reg_off.off == 0, + else => false, + }; + } + + fn isRegisterOf(mcv: MCValue, rc: Register.Class) bool { + return switch (mcv) { + .register => |reg| reg.class() == rc, + .register_bias => |reg_off| reg_off.off == 0 and reg_off.reg.class() == rc, + else => false, + }; + } + + fn isInRegister(mcv: MCValue) bool { + return switch (mcv) { + .register, .register_pair, .register_triple, .register_quadruple => true, + .register_bias => |reg_off| reg_off.off == 0, + else => false, + }; + } + + fn isRegisterBias(mcv: MCValue) bool { + return switch (mcv) { + .register, .register_bias => true, + else => false, + }; + } + + fn getReg(mcv: MCValue) ?Register { + return switch (mcv) { + .register => |reg| reg, + .register_bias, .register_offset => |ro| ro.reg, + .register_frame => |reg_frame| reg_frame.reg, + else => null, + }; + } + + fn getRegs(mcv: *const MCValue) []const Register { + return switch (mcv.*) { + .register => |*reg| reg[0..1], + inline .register_pair, + .register_triple, + .register_quadruple, + => |*regs| regs, + inline .register_bias, + .register_offset, + => |*pl| (&pl.reg)[0..1], + .register_frame => |*reg_frame| (®_frame.reg)[0..1], + else => &.{}, + }; + } + + fn isAddress(mcv: MCValue) bool { + return switch (mcv) { + .immediate, .register, .register_bias, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => true, + else => false, + }; + } + + fn address(mcv: MCValue) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_bias, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_frame, + .reserved_frame, + .air_ref, + => unreachable, // not in memory + .memory => |addr| .{ .immediate = addr }, + .register_offset => |reg_off| switch (reg_off.off) { + 0 => .{ .register = reg_off.reg }, + else => .{ .register_bias = reg_off }, + }, + .load_frame => |frame_addr| .{ .lea_frame = frame_addr }, + .register_frame => |reg_frame| .{ .lea_frame = reg_frame.frame }, + .load_nav => |nav_off| .{ .lea_nav = nav_off }, + .load_uav => |uav_off| .{ .lea_uav = uav_off }, + .load_lazy_sym => |sym| .{ .lea_lazy_sym = sym }, + }; + } + + fn deref(mcv: MCValue) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .register_pair, + .register_triple, + .register_quadruple, + .memory, + .register_offset, + .load_frame, + .register_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .reserved_frame, + .air_ref, + => unreachable, // not dereferenceable + .immediate => |addr| .{ .memory = addr }, + .register => |reg| .{ .register_offset = .{ .reg = reg } }, + .register_bias => |reg_off| .{ .register_offset = reg_off }, + .lea_frame => |frame_addr| .{ .load_frame = frame_addr }, + .lea_nav => |nav_off| .{ .load_nav = nav_off }, + .lea_uav => |uav_off| .{ .load_uav = uav_off }, + .lea_lazy_sym => |sym| .{ .load_lazy_sym = sym }, + }; + } + + fn offset(mcv: MCValue, off: i32) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .reserved_frame, + .air_ref, + => unreachable, // not valid + .register_pair, + .register_triple, + .register_quadruple, + .memory, + .register_offset, + .load_frame, + .register_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .lea_lazy_sym, + => switch (off) { + 0 => mcv, + else => unreachable, // not offsettable + }, + .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, + .register => |reg| .{ .register_bias = .{ .reg = reg, .off = off } }, + .register_bias => |reg_off| .{ + .register_bias = .{ .reg = reg_off.reg, .off = reg_off.off + off }, + }, + .lea_frame => |frame_addr| .{ + .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, + }, + .lea_nav => |nav_off| .{ + .lea_nav = .{ .index = nav_off.index, .off = nav_off.off + off }, + }, + .lea_uav => |uav_off| .{ + .lea_uav = .{ .index = uav_off.index, .off = uav_off.off + off }, + }, + }; + } + + /// Returns MCV of a limb. + /// Caller does not own returned values. + fn toLimbValue(mcv: MCValue, limb_index: u64) MCValue { + switch (mcv) { + else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), + .register, .immediate, .register_bias, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => { + assert(limb_index == 0); + return mcv; + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + return .{ .register = regs[@intCast(limb_index)] }; + }, + .load_frame => |frame_addr| { + return .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(u31, @intCast(limb_index)) * 8, + } }; + }, + .register_offset => |reg_off| { + return .{ .register_offset = .{ + .reg = reg_off.reg, + .off = reg_off.off + @as(u31, @intCast(limb_index)) * 8, + } }; + }, + .load_nav => |nav_off| { + return .{ .load_nav = .{ + .index = nav_off.index, + .off = nav_off.off + @as(u31, @intCast(limb_index)) * 8, + } }; + }, + .load_uav => |uav_off| { + return .{ .load_uav = .{ + .index = uav_off.index, + .off = uav_off.off + @as(u31, @intCast(limb_index)) * 8, + } }; + }, + } + } + + pub fn format( + mcv: MCValue, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (mcv) { + .none, .unreach, .dead, .undef => try writer.print("({s})", .{@tagName(mcv)}), + .immediate => |pl| try writer.print("0x{x}", .{pl}), + .register => |pl| try writer.print("{s}", .{@tagName(pl)}), + .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), + .register_triple => |pl| try writer.print("{s}:{s}:{s}", .{ + @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), + }), + .register_quadruple => |pl| try writer.print("{s}:{s}:{s}:{s}", .{ + @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), + }), + .register_bias => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), + .memory => |pl| try writer.print("[0x{x}]", .{pl}), + .load_frame => |pl| try writer.print("[frame:{} + 0x{x}]", .{ pl.index, pl.off }), + .lea_frame => |pl| try writer.print("frame:{} + 0x{x}", .{ pl.index, pl.off }), + .register_frame => |pl| try writer.print("{{{s}, (frame:{} + 0x{x})}}", .{ @tagName(pl.reg), pl.frame.index, pl.frame.off }), + .register_offset => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), + .load_nav => |pl| try writer.print("[nav:{} + 0x{x}]", .{ pl.index, pl.off }), + .lea_nav => |pl| try writer.print("nav:{} + 0x{x}", .{ pl.index, pl.off }), + .load_uav => |pl| try writer.print("[uav:{} + 0x{x}]", .{ pl.index, pl.off }), + .lea_uav => |pl| try writer.print("uav:{} + 0x{x}", .{ pl.index, pl.off }), + .load_lazy_sym => |pl| try writer.print("[lazy sym:{s}:{}]", .{ @tagName(pl.kind), pl.ty }), + .lea_lazy_sym => |pl| try writer.print("lazy sym:{s}:{}", .{ @tagName(pl.kind), pl.ty }), + .reserved_frame => |pl| try writer.print("(dead:{})", .{pl}), + .air_ref => |pl| try writer.print("(air:{})", .{@intFromEnum(pl)}), + } + } + + /// Converts a CCV to MCV. + /// `ref_frame` values will be converted into `load_frame` + /// pointing to the ptr on the call frame as MCValue cannot + /// represent double indirect. + fn fromCCValue(ccv: abi.CCValue, frame: FrameIndex) MCValue { + return switch (ccv) { + .none => .none, + .register => |reg| .{ .register = reg }, + .register_pair => |regs| .{ .register_pair = regs }, + .register_triple => |regs| .{ .register_triple = regs }, + .register_quadruple => |regs| .{ .register_quadruple = regs }, + .frame => |off| .{ .load_frame = .{ .index = frame, .off = off } }, + .split => |pl| .{ .register_frame = .{ + .reg = pl.reg, + .frame = .{ .index = frame, .off = pl.frame_off }, + } }, + .ref_register => |reg| .{ .register_offset = .{ .reg = reg, .off = 0 } }, + .ref_frame => |off| .{ .load_frame = .{ .index = frame, .off = off } }, + }; + } + + /// Allocates and loads `val` to registers if `val` is not in regs yet. + /// Returns either self or the allocated reg and a flag indicating whether allocation happened. + fn ensureReg(val: MCValue, cg: *CodeGen, inst: Air.Inst.Index, ty: Type) InnerError!struct { Register, bool } { + if (val.isInRegister()) { + return .{ val.register, false }; + } else { + const temp_reg = try cg.allocReg(ty, inst); + try cg.genCopyToReg(.fromByteSize(ty.abiSize(cg.pt.zcu)), temp_reg, val, .{}); + return .{ temp_reg, true }; + } + } +}; + +const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking); +const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking); + +const InstTracking = struct { + long: MCValue, + short: MCValue, + + fn init(result: MCValue) InstTracking { + return .{ .long = switch (result) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + => result, + .dead, + .reserved_frame, + .air_ref, + => unreachable, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_bias, + .register_offset, + .register_frame, + => .none, + }, .short = result }; + } + + fn getReg(inst_tracking: InstTracking) ?Register { + return inst_tracking.short.getReg(); + } + + fn getRegs(inst_tracking: *const InstTracking) []const Register { + return inst_tracking.short.getRegs(); + } + + pub fn format( + inst_tracking: InstTracking, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + if (!std.meta.eql(inst_tracking.long, inst_tracking.short)) + try writer.print("|{}| ", .{inst_tracking.long}); + try writer.print("{}", .{inst_tracking.short}); + } + + fn die(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { + if (self.short == .dead) return; + try cg.freeValue(self.short); + + if (self.long == .none) { + switch (self.short) { + .load_frame => |frame| { + if (frame.off == 0) self.long = .{ .reserved_frame = frame.index }; + }, + else => {}, + } + } + + self.short = .{ .dead = cg.scope_generation }; + tracking_log.debug("{} => {} (death)", .{ inst, self.* }); + } + + fn reuse( + self: *InstTracking, + cg: *CodeGen, + new_inst: Air.Inst.Index, + old_inst: Air.Inst.Index, + ) void { + tracking_log.debug("{} => {} (reuse {})", .{ new_inst, self.*, old_inst }); + self.short = .{ .dead = cg.scope_generation }; + } + + fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { + if (std.meta.eql(self.long, self.short)) return; // Already spilled + // Allocate or reuse frame index + switch (self.long) { + .none => self.long = try cg.allocRegOrMem(cg.typeOfIndex(inst), inst, .{ .use_reg = false }), + .load_frame => {}, + .lea_frame => return, + .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, + else => unreachable, + } + tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long }); + try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{}); + } + + fn trackSpill(inst_tracking: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { + try cg.freeValue(inst_tracking.short); + inst_tracking.reuseFrame(); + tracking_log.debug("{} => {} (spilled)", .{ inst, inst_tracking.* }); + } + + fn reuseFrame(inst_tracking: *InstTracking) void { + inst_tracking.* = .init(switch (inst_tracking.long) { + .none => switch (inst_tracking.short) { + .dead => .none, + else => |short| short, + }, + .reserved_frame => |index| .{ .load_frame = .{ .index = index } }, + else => |long| long, + }); + } + + fn resurrect(inst_tracking: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index, scope_generation: u32) !void { + switch (inst_tracking.short) { + .dead => |die_generation| if (die_generation >= scope_generation) { + inst_tracking.reuseFrame(); + try cg.getValue(inst_tracking.short, inst); + tracking_log.debug("{} => {} (resurrect)", .{ inst, inst_tracking.* }); + }, + else => {}, + } + } + + fn verifyMaterialize(inst_tracking: InstTracking, target: InstTracking) void { + switch (inst_tracking.long) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + => assert(std.meta.eql(inst_tracking.long, target.long)), + .load_frame, + .reserved_frame, + => switch (target.long) { + .none, + .load_frame, + .reserved_frame, + => {}, + else => unreachable, + }, + else => unreachable, + } + } + + fn materialize(inst_tracking: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index, target: InstTracking) !void { + inst_tracking.verifyMaterialize(target); + try inst_tracking.materializeUnsafe(cg, inst, target); + } + + fn materializeUnsafe(inst_tracking: InstTracking, cg: *CodeGen, inst: Air.Inst.Index, target: InstTracking) !void { + const ty = cg.typeOfIndex(inst); + if ((inst_tracking.long == .none or inst_tracking.long == .reserved_frame) and target.long == .load_frame) + try cg.genCopy(ty, target.long, inst_tracking.short, .{}); + try cg.genCopy(ty, target.short, inst_tracking.short, .{}); + } + + fn trackMaterialize(inst_tracking: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void { + inst_tracking.verifyMaterialize(target); + // do not clobber reserved frame indices + inst_tracking.long = if (target.long == .none) switch (inst_tracking.long) { + .load_frame => |addr| .{ .reserved_frame = addr.index }, + .reserved_frame => inst_tracking.long, + else => target.long, + } else target.long; + inst_tracking.short = target.short; + tracking_log.debug("{} => {} (materialize)", .{ inst, inst_tracking.* }); + } + + fn liveOut(inst_tracking: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) void { + for (inst_tracking.getRegs()) |reg| { + if (cg.register_manager.isRegFree(reg)) { + tracking_log.debug("{} => {} (live-out)", .{ inst, inst_tracking.* }); + continue; + } + + const index = RegisterManager.indexOfRegIntoTracked(reg).?; + const tracked_inst = cg.register_manager.registers[index]; + const tracking = cg.resolveInst(tracked_inst); + + // Disable death. + var found_reg = false; + var remaining_reg: Register = .zero; + for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) { + assert(!found_reg); + found_reg = true; + } else { + assert(remaining_reg == Register.zero); + remaining_reg = tracked_reg; + }; + assert(found_reg); + tracking.short = switch (remaining_reg) { + Register.zero => .{ .dead = cg.scope_generation }, + else => .{ .register = remaining_reg }, + }; + + // Perform side-effects of freeValue manually. + cg.register_manager.freeReg(reg); + + tracking_log.debug("{} => {} (live-out {})", .{ inst, inst_tracking.*, tracked_inst }); + } + } +}; + +const FrameAlloc = struct { + abi_size: u31, + spill_pad: u3, + abi_align: InternPool.Alignment, + + fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc { + return .{ + .abi_size = @intCast(alloc_abi.size), + .spill_pad = alloc_abi.pad, + .abi_align = alloc_abi.alignment, + }; + } + + fn initType(ty: Type, zcu: *Zcu) FrameAlloc { + return init(.{ + .size = ty.abiSize(zcu), + .alignment = ty.abiAlignment(zcu), + }); + } +}; + +pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { + return comptime &.initMany(&.{ + .expand_intcast_safe, + .expand_add_safe, + .expand_sub_safe, + .expand_mul_safe, + .expand_packed_load, + .expand_packed_store, + .expand_packed_struct_field_val, + .expand_packed_aggregate_init, + }); +} + +pub fn generate( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + air: *const Air, + liveness: *const Air.Liveness, +) codegen.CodeGenError!Mir { + _ = bin_file; + const zcu = pt.zcu; + const gpa = zcu.gpa; + const ip = &zcu.intern_pool; + const func = zcu.funcInfo(func_index); + const fn_type: Type = .fromInterned(func.ty); + const mod = zcu.navFileScope(func.owner_nav).mod.?; + + // initialize CG + var cg: CodeGen = .{ + .gpa = gpa, + .pt = pt, + .air = air.*, + .liveness = liveness.*, + .target = &mod.resolved_target.result, + .mod = mod, + .owner = .{ .nav_index = func.owner_nav }, + .inline_func = func_index, + .arg_index = 0, + .call_info = undefined, + .args_mcv = undefined, + .ret_mcv = undefined, + .fn_type = fn_type, + .src_loc = src_loc, + }; + try cg.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); + for (0..Temp.Index.max) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + cg.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none)); + } + defer { + cg.frame_allocs.deinit(gpa); + cg.free_frame_indices.deinit(gpa); + cg.frame_locs.deinit(gpa); + cg.loops.deinit(gpa); + cg.blocks.deinit(gpa); + cg.inst_tracking.deinit(gpa); + cg.mir_instructions.deinit(gpa); + } + + cg_mir_log.debug("{}:", .{ip.getNav(func.owner_nav).fqn.fmt(ip)}); + + // resolve CC values + const fn_ty = zcu.typeToFunc(fn_type).?; + cg.call_info = try cg.resolveCallInfo(&fn_ty); + defer cg.call_info.deinit(gpa); + + if (cg.call_info.err_ret_trace_reg) |err_ret_trace_reg| { + cg.register_manager.getRegAssumeFree(err_ret_trace_reg, err_ret_trace_index); + try cg.inst_tracking.putNoClobber( + gpa, + err_ret_trace_index, + .init(.{ .register = err_ret_trace_reg }), + ); + } + + cg.args_mcv = try gpa.alloc(MCValue, cg.call_info.params.len); + defer gpa.free(cg.args_mcv); + for (cg.call_info.params, 0..) |ccv, i| { + cg.args_mcv[i] = .fromCCValue(ccv, .args_frame); + for (ccv.getRegs()) |arg_reg| { + cg.register_manager.getRegAssumeFree(arg_reg, null); + } + } + cg.ret_mcv = .fromCCValue(cg.call_info.return_value, .args_frame); + switch (cg.call_info.return_value) { + .ref_register => |reg| cg.register_manager.getRegAssumeFree(reg, null), + else => {}, + } + + // init basic frames + try cg.frame_allocs.resize(gpa, FrameIndex.named_count); + inline for ([_]FrameIndex{ + .stack_frame, + .call_frame, + .spill_int_frame, + .spill_float_frame, + .ret_addr_frame, + }) |frame| { + cg.frame_allocs.set( + @intFromEnum(frame), + .init(.{ .size = 0, .alignment = .@"1" }), + ); + } + cg.frame_allocs.set( + @intFromEnum(FrameIndex.args_frame), + .init(.{ + .size = cg.call_info.frame_size, + .alignment = cg.call_info.frame_align, + }), + ); + + // generate code + cg.gen() catch |err| switch (err) { + error.OutOfRegisters => return cg.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + // end the function at the right brace + if (!cg.mod.strip) _ = try cg.addInst(.{ + .tag = .fromPseudo(.dbg_line_line_column), + .data = .{ .line_column = .{ + .line = func.rbrace_line, + .column = func.rbrace_column, + } }, + }); + + // check if $ra needs to be spilled + const ra_allocated = cg.register_manager.isRegAllocated(.ra); + if (ra_allocated) { + cg.frame_allocs.set( + @intFromEnum(FrameIndex.ret_addr_frame), + .init(.{ .size = 8, .alignment = .@"8" }), + ); + } + + // compute frame layout + const frame_size = try cg.computeFrameLayout(); + log.debug("Frame layout: {} bytes{}", .{ frame_size, cg.fmtFrameLocs() }); + + // construct MIR + const mir: Mir = .{ + .instructions = cg.mir_instructions.toOwnedSlice(), + .frame_locs = cg.frame_locs.toOwnedSlice(), + .frame_size = frame_size, + .epilogue_begin = cg.epilogue_label, + .spill_ra = ra_allocated, + }; + + return mir; +} + +pub fn generateLazy( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + lazy_sym: link.File.LazySymbol, + code: *std.ArrayListUnmanaged(u8), + debug_output: link.File.DebugInfoOutput, +) codegen.CodeGenError!void { + _ = bin_file; + _ = pt; + _ = src_loc; + _ = lazy_sym; + _ = code; + _ = debug_output; + + unreachable; +} + +/// Computes frame layout and fill `frame_locs` based on `frame_allocs`. +/// Returns size of the frame. +fn computeFrameLayout(cg: *CodeGen) !usize { + const frame_allocs_len = cg.frame_allocs.len; + try cg.frame_locs.resize(cg.gpa, frame_allocs_len); + + const frame_align = cg.frame_allocs.items(.abi_align); + const frame_offset = cg.frame_locs.items(.offset); + + // sort frames by alignment + const frame_alloc_order = try cg.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count); + defer cg.gpa.free(frame_alloc_order); + + for (frame_alloc_order, FrameIndex.named_count..) |*frame_order, frame_index| + frame_order.* = @enumFromInt(frame_index); + + { + const SortContext = struct { + frame_align: @TypeOf(frame_align), + pub fn lessThan(ctx: @This(), lhs: FrameIndex, rhs: FrameIndex) bool { + return ctx.frame_align[@intFromEnum(lhs)].compare(.gt, ctx.frame_align[@intFromEnum(rhs)]); + } + }; + const sort_context: SortContext = .{ .frame_align = frame_align }; + std.mem.sort(FrameIndex, frame_alloc_order, sort_context, SortContext.lessThan); + } + + // TODO: use frame pointer when needed + // TODO: optimize: don't touch sp for leave functions + + // compute locations from the bottom to the top + var sp_offset: i32 = 0; + cg.setFrameLoc(.stack_frame, .sp, &sp_offset, true); + cg.setFrameLoc(.call_frame, .sp, &sp_offset, true); + for (frame_alloc_order) |frame_index| cg.setFrameLoc(frame_index, .sp, &sp_offset, true); + cg.setFrameLoc(.spill_float_frame, .sp, &sp_offset, true); + cg.setFrameLoc(.spill_int_frame, .sp, &sp_offset, true); + cg.setFrameLoc(.ret_addr_frame, .sp, &sp_offset, true); + cg.setFrameLoc(.args_frame, .sp, &sp_offset, false); + + return @intCast(sp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]); +} + +fn setFrameLoc(cg: *CodeGen, frame_index: FrameIndex, base: Register, offset: *i32, comptime aligned: bool) void { + const frame_i = @intFromEnum(frame_index); + if (aligned) { + const alignment = cg.frame_allocs.items(.abi_align)[frame_i]; + offset.* = @intCast(alignment.forward(@intCast(offset.*))); + } + cg.frame_locs.set(frame_i, .{ .base = base, .offset = offset.* }); + offset.* += cg.frame_allocs.items(.abi_size)[frame_i]; +} + +/// Adjusts a frame alloc for a required size and alignment. +fn adjustFrame(cg: *CodeGen, frame: FrameIndex, size: u32, alignment: InternPool.Alignment) void { + const frame_size = + &cg.frame_allocs.items(.abi_size)[@intFromEnum(frame)]; + frame_size.* = @max(frame_size.*, @as(u31, @intCast(size))); + const frame_align = + &cg.frame_allocs.items(.abi_align)[@intFromEnum(frame)]; + frame_align.* = frame_align.max(alignment); +} + +/// Computes static registers that need to be spilled. +fn computeSpillRegs(cg: *CodeGen, regs: []const Register) Mir.RegisterList { + var list: Mir.RegisterList = .empty; + for (regs) |reg| + if (cg.register_manager.isRegAllocated(reg)) list.push(reg); + return list; +} + +/// Adjusts spill frames. +fn adjustSpillFrame(cg: *CodeGen, rc: Register.Class, regs: Mir.RegisterList) void { + const size = rc.byteSize(cg.target) * regs.count(); + const frame: FrameIndex = switch (rc) { + .int => .spill_int_frame, + .float => .spill_float_frame, + else => unreachable, + }; + cg.adjustFrame(frame, @intCast(size), .fromByteUnits(@intCast(rc.byteSize(cg.target)))); +} + +/// Generates the function body. +fn gen(cg: *CodeGen) InnerError!void { + try cg.asmPseudo(.func_prologue, .none); + const bp_spill_regs_int = try cg.asmPlaceholder(); + const bp_spill_regs_float = try cg.asmPlaceholder(); + + try cg.genBody(cg.air.getMainBody()); + cg.epilogue_label = cg.label(); + + // Spill static registers + const spill_regs_int = cg.computeSpillRegs(&(abi.zigcc.Integer.static_regs)); + const spill_regs_float = cg.computeSpillRegs(&abi.zigcc.Floating.static_regs); + cg.backpatchPseudo(bp_spill_regs_int, .spill_int_regs, .{ .reg_list = spill_regs_int }); + cg.backpatchPseudo(bp_spill_regs_float, .spill_float_regs, .{ .reg_list = spill_regs_float }); + try cg.asmPseudo(.restore_int_regs, .{ .reg_list = spill_regs_int }); + try cg.asmPseudo(.restore_float_regs, .{ .reg_list = spill_regs_float }); + cg.adjustSpillFrame(.int, spill_regs_int); + cg.adjustSpillFrame(.float, spill_regs_float); + + try cg.asmPseudo(.func_epilogue, .none); +} + +/// Generates a lexical block. +fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { + if (!cg.mod.strip) try cg.asmPseudo(.dbg_enter_block, .none); + try cg.genBody(body); + if (!cg.mod.strip) try cg.asmPseudo(.dbg_exit_block, .none); +} + +fn hasFeature(cg: *CodeGen, feature: std.Target.loongarch.Feature) bool { + return cg.target.cpu.has(.loongarch, feature); +} + +fn intRegisterSize(cg: *CodeGen) u16 { + return switch (cg.target.cpu.arch) { + .loongarch32 => 32, + .loongarch64 => 64, + else => unreachable, + }; +} + +fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = cg.pt.zcu; + switch (cg.owner) { + .nav_index => |i| return zcu.codegenFail(i, format, args), + .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args), + } + return error.CodegenFail; +} + +fn failMsg(self: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = self.pt.zcu; + switch (self.owner) { + .nav_index => |i| return zcu.codegenFailMsg(i, msg), + .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg), + } + return error.CodegenFail; +} + +fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { + const gpa = cg.gpa; + + try cg.mir_instructions.ensureUnusedCapacity(gpa, 1); + const result_index: Mir.Inst.Index = @intCast(cg.mir_instructions.len); + + cg_mir_log.debug(" | {}: {}", .{ result_index, inst }); + + cg.mir_instructions.appendAssumeCapacity(inst); + return result_index; +} + +fn backpatchInst(cg: *CodeGen, index: Mir.Inst.Index, inst: Mir.Inst) void { + cg_mir_log.debug(" | backpatch: {}: {}", .{ index, inst }); + cg.mir_instructions.set(index, inst); +} + +inline fn asmPseudo(cg: *CodeGen, tag: Mir.Inst.PseudoTag, data: Mir.Inst.Data) error{OutOfMemory}!void { + _ = try cg.addInst(.{ .tag = .fromPseudo(tag), .data = data }); +} + +inline fn asmPlaceholder(cg: *CodeGen) error{OutOfMemory}!u32 { + return cg.addInst(.{ .tag = .fromPseudo(.none), .data = .none }); +} + +inline fn backpatchPseudo(cg: *CodeGen, index: Mir.Inst.Index, tag: Mir.Inst.PseudoTag, data: Mir.Inst.Data) void { + cg.backpatchInst(index, .{ .tag = .fromPseudo(tag), .data = data }); +} + +inline fn asmInst(cg: *CodeGen, inst: encoding.Inst) error{OutOfMemory}!void { + _ = try cg.addInst(.initInst(inst)); +} + +fn asmBr(cg: *CodeGen, target: ?Mir.Inst.Index, cond: Mir.BranchCondition) InnerError!Mir.Inst.Index { + return cg.addInst(.{ + .tag = .fromPseudo(.branch), + .data = .{ + .br = .{ + .inst = target orelse 0, // use zero instead of undefined to make debugging easier + .cond = cond, + }, + }, + }); +} + +fn performReloc(cg: *CodeGen, reloc: Mir.Inst.Index) void { + cg_mir_log.debug(" | <-- reloc {}", .{reloc}); + + const next_inst: u32 = @intCast(cg.mir_instructions.len); + switch (cg.mir_instructions.items(.tag)[reloc].unwrap()) { + .inst => unreachable, + .pseudo => |tag| { + switch (tag) { + .branch => cg.mir_instructions.items(.data)[reloc].br.inst = next_inst, + else => unreachable, + } + }, + } +} + +fn label(cg: *CodeGen) Mir.Inst.Index { + return @intCast(cg.mir_instructions.len); +} + +/// A temporary operand. +/// Either a allocated temp or a Air.Inst.Ref (ref to a AIR inst or a interned val) or error return trace. +const Temp = struct { + index: Air.Inst.Index, + + const Index = enum(u5) { + _, + + fn toIndex(index: Index) Air.Inst.Index { + return .fromTargetIndex(@intFromEnum(index)); + } + + fn fromIndex(index: Air.Inst.Index) Index { + return @enumFromInt(index.toTargetIndex()); + } + + fn tracking(index: Index, cg: *CodeGen) *InstTracking { + return &cg.inst_tracking.values()[@intFromEnum(index)]; + } + + fn isValid(index: Index, cg: *CodeGen) bool { + return @intFromEnum(index) < @intFromEnum(cg.next_temp_index) and + index.tracking(cg).short != .dead; + } + + fn typeOf(index: Index, cg: *CodeGen) Type { + assert(index.isValid(cg)); + return cg.temp_type[@intFromEnum(index)]; + } + + fn toType(index: Index, cg: *CodeGen, ty: Type) void { + assert(index.isValid(cg)); + cg.temp_type[@intFromEnum(index)] = ty; + } + + const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type); + }; + + fn unwrap(temp: Temp, cg: *CodeGen) union(enum) { ref: Air.Inst.Ref, temp: Index, err_ret_trace } { + switch (temp.index.unwrap()) { + .ref => |ref| return .{ .ref = ref }, + .target => |target_index| { + if (@as(Air.Inst.Index, @enumFromInt(target_index)) == err_ret_trace_index) return .err_ret_trace; + const temp_index: Index = @enumFromInt(target_index); + assert(temp_index.isValid(cg)); + return .{ .temp = temp_index }; + }, + } + } + + fn typeOf(temp: Temp, cg: *CodeGen) Type { + return switch (temp.unwrap(cg)) { + .ref => switch (cg.air.instructions.items(.tag)[@intFromEnum(temp.index)]) { + .loop_switch_br => cg.typeOf(cg.air.unwrapSwitch(temp.index).operand), + else => cg.air.typeOfIndex(temp.index, &cg.pt.zcu.intern_pool), + }, + .temp => |temp_index| temp_index.typeOf(cg), + .err_ret_trace => .usize, + }; + } + + fn tracking(temp: Temp, cg: *CodeGen) InstTracking { + return cg.inst_tracking.get(temp.index).?; + } + + fn maybeTracking(temp: Temp, cg: *CodeGen) ?InstTracking { + switch (temp.index.unwrap()) { + .ref => |temp_ref| if (temp_ref.toInternedAllowNone() != null) return null, + else => {}, + } + return cg.inst_tracking.get(temp.index).?; + } + + fn isMut(temp: Temp, cg: *CodeGen) bool { + return switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => false, + .temp => |_| temp.tracking(cg).short.isModifiable(), + }; + } + + fn toMemory(temp: *Temp, cg: *CodeGen) InnerError!bool { + switch (temp.tracking(cg).short) { + .none, + .unreach, + .dead, + .undef, + .register_pair, + .register_triple, + .register_quadruple, + .reserved_frame, + .air_ref, + => unreachable, // not a valid pointer + .immediate, + .register, + .register_bias, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + => { + const ty = temp.typeOf(cg); + const new_temp = try cg.tempAllocMem(ty); + temp.copy(new_temp, cg, ty); + try temp.die(cg); + temp.* = new_temp; + return true; + }, + .memory, + .register_offset, + .load_frame, + .register_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + => return false, + } + } + + fn getReg(temp: Temp, cg: *CodeGen) Register { + return temp.tracking(cg).getReg().?; + } + + fn getUnsignedImm(temp: Temp, cg: *CodeGen) u64 { + return temp.tracking(cg).short.immediate; + } + + fn getSignedImm(temp: Temp, cg: *CodeGen) i64 { + return @bitCast(temp.tracking(cg).short.immediate); + } + + fn moveToMemory(temp: *Temp, cg: *CodeGen, mut: bool) InnerError!bool { + const temp_mcv = temp.tracking(cg).short; + if (temp_mcv.isMemory() and (temp_mcv.isModifiable() or !mut)) { + return false; + } else { + const temp_ty = temp.typeOf(cg); + try temp.die(cg); + temp.* = try cg.tempAllocMem(temp_ty); + try cg.genCopy(temp_ty, temp.tracking(cg).short, temp_mcv, .{}); + return true; + } + } + + fn moveToRegister(temp: *Temp, cg: *CodeGen, rc: Register.Class, mut: bool) InnerError!bool { + const temp_mcv = temp.tracking(cg).short; + if (temp.tracking(cg).short.isRegister() and (temp_mcv.isModifiable() or !mut)) { + return false; + } else { + const old_mcv = temp.tracking(cg).short; + const temp_ty = temp.typeOf(cg); + try temp.die(cg); + temp.* = try cg.tempAllocReg(temp_ty, abi.getAllocatableRegSet(rc)); + try cg.genCopy(temp_ty, temp.tracking(cg).short, old_mcv, .{}); + return true; + } + } + + fn die(temp: Temp, cg: *CodeGen) InnerError!void { + switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => {}, + .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex()), + } + } + + fn finish( + temp: Temp, + inst: Air.Inst.Index, + ops: []const Temp, + cg: *CodeGen, + ) InnerError!void { + const tomb_bits = cg.liveness.getTombBits(inst); + // Tomb operands except for that are used as result + for (0.., ops) |op_index, op| { + if (op.index == temp.index) continue; + if (op.tracking(cg).short != .dead) try op.die(cg); + if ((tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 1) and !cg.reused_operands.isSet(op_index)) { + switch (temp.index.unwrap()) { + .ref => |op_ref| if (op_ref.toIndex()) |op_inst| { + try cg.inst_tracking.getPtr(op_inst).?.die(cg, op_inst); + }, + .target => {}, + } + } + } + if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => { + const ty = cg.typeOfIndex(inst); + const result_mcv = try cg.allocRegOrMem(ty, inst, .{}); + try cg.genCopy(cg.typeOfIndex(inst), result_mcv, temp.tracking(cg).short, .{}); + + tracking_log.debug("{} => {} (birth copied from {})", .{ inst, result_mcv, temp.index }); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result_mcv)); + }, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + tracking_log.debug("{} => {} (birth from {})", .{ inst, temp_tracking.short, temp.index }); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(temp_tracking.short)); + assert(cg.reuseTemp(temp_tracking, inst, temp_index.toIndex())); + }, + } + // Tomb operands that are used as result + for (0.., ops) |op_index, op| { + if (op.index != temp.index) continue; + if ((tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 1) and !cg.reused_operands.isSet(op_index)) { + switch (temp.index.unwrap()) { + .ref => |op_ref| if (op_ref.toIndex()) |op_inst| { + try cg.inst_tracking.getPtr(op_inst).?.die(cg, op_inst); + }, + .target => {}, + } + } + } + } + + fn copy(src: Temp, dst: Temp, cg: *CodeGen, ty: Type) InnerError!void { + try cg.genCopy(ty, dst.tracking(cg).short, src.tracking(cg).short, .{}); + } + + const AccessOptions = struct { + safety: bool = false, + off: i32 = 0, + }; + + fn load(ptr: Temp, cg: *CodeGen, ty: Type, opts: AccessOptions) InnerError!Temp { + const val = try cg.tempAlloc(ty, .{}); + try ptr.loadTo(val, cg, ty, opts); + return val; + } + + fn loadTo(ptr: Temp, dst: Temp, cg: *CodeGen, ty: Type, opts: AccessOptions) InnerError!void { + const addr, _ = try ptr.ensureOffsetable(cg); + addr.toOffset(cg, opts.off); + try cg.genCopy(ty, dst.tracking(cg).short, addr.tracking(cg).short.deref(), .{ .safety = opts.safety }); + } + + fn storeTo(val: Temp, ptr: Temp, cg: *CodeGen, ty: Type, opts: AccessOptions) InnerError!void { + const addr, _ = try ptr.ensureOffsetable(cg); + addr.toOffset(cg, opts.off); + try cg.genCopy(ty, addr.tracking(cg).short.deref(), val.tracking(cg).short, .{ .safety = opts.safety }); + } + + fn read(val: Temp, cg: *CodeGen, ty: Type, opts: AccessOptions) InnerError!Temp { + const dst = try cg.tempAlloc(ty, .{}); + try val.readTo(dst, cg, ty, opts); + return dst; + } + + fn readTo(val: Temp, dst: Temp, cg: *CodeGen, ty: Type, opts: AccessOptions) InnerError!void { + const val_mcv = val.tracking(cg).short; + switch (val_mcv) { + else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), + .register, .register_bias => { + assert(opts.off == 0); + try val.copy(dst, cg, ty); + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + assert(@rem(opts.off, 8) == 0); + assert(opts.off >= 0); + assert(ty.abiSize(cg.pt.zcu) <= 8); // TODO not implemented yet + try cg.genCopy(ty, dst.tracking(cg).short, .{ .register = regs[@as(u32, @intCast(opts.off)) / 8] }, .{ .safety = opts.safety }); + }, + .memory, .register_offset, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var addr_ptr = try cg.tempInit(.usize, val_mcv.address()); + try addr_ptr.loadTo(dst, cg, ty, opts); + try addr_ptr.die(cg); + }, + } + } + + fn write(dst: Temp, val: Temp, cg: *CodeGen, opts: AccessOptions) InnerError!void { + const ty = val.typeOf(cg); + const dst_mcv = dst.tracking(cg).short; + switch (dst_mcv) { + else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), + .register, .register_bias => { + assert(opts.off == 0); + try val.copy(dst, cg, ty); + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + assert(@rem(opts.off, 8) == 0); + assert(opts.off >= 0); + assert(ty.abiSize(cg.pt.zcu) <= 8); // TODO not implemented yet + try cg.genCopy(ty, dst.tracking(cg).short, .{ .register = regs[@as(u32, @intCast(opts.off)) / 8] }, .{ .safety = opts.safety }); + }, + .memory, .register_offset, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var addr_ptr = try cg.tempInit(.usize, dst_mcv.address()); + try val.storeTo(addr_ptr, cg, ty, opts); + try addr_ptr.die(cg); + }, + } + } + + fn getLimbCount(temp: Temp, cg: *CodeGen) u64 { + return cg.getLimbCount(temp.typeOf(cg)); + } + + fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen, reuse: bool) InnerError!Temp { + if (reuse) try temp.die(cg); + const new_temp_index = cg.next_temp_index; + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty; + const temp_mcv = temp.tracking(cg).short; + switch (temp_mcv) { + else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), + .immediate => |imm| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); + }, + .register => |reg| { + assert(limb_index == 0); + if (reuse) + new_temp_index.tracking(cg).* = .init(.{ .register = reg }) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmInst(.@"or"(new_reg, reg, .zero)); + } + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + if (reuse) + new_temp_index.tracking(cg).* = .init(.{ .register = regs[@intCast(limb_index)] }) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmInst(.@"or"(new_reg, regs[@intCast(limb_index)], .zero)); + } + }, + .register_bias, .register_offset => |_| { + assert(limb_index == 0); + if (reuse) + new_temp_index.tracking(cg).* = .init(temp_mcv) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.genCopyToReg(.dword, new_reg, temp.tracking(cg).short, .{}); + } + }, + .load_frame => |frame_addr| { + const new_mcv: MCValue = .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(u31, limb_index) * 8, + } }; + if (reuse) + new_temp_index.tracking(cg).* = .init(new_mcv) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.genCopyToReg(.dword, new_reg, new_mcv, .{}); + } + }, + .lea_frame => |frame_addr| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr }); + }, + .load_nav => |nav_off| { + const new_mcv: MCValue = .{ .load_nav = .{ + .index = nav_off.index, + .off = nav_off.off + @as(u31, limb_index) * 8, + } }; + if (reuse) + new_temp_index.tracking(cg).* = .init(new_mcv) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.genCopyToReg(.dword, new_reg, new_mcv, .{}); + } + }, + .lea_nav => |nav_off| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_nav = nav_off }); + }, + .load_uav => |uav_off| { + const new_mcv: MCValue = .{ .load_uav = .{ + .index = uav_off.index, + .off = uav_off.off + @as(u31, limb_index) * 8, + } }; + if (reuse) + new_temp_index.tracking(cg).* = .init(new_mcv) + else { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterSets.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.genCopyToReg(.dword, new_reg, new_mcv, .{}); + } + }, + .lea_uav => |nav_off| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_uav = nav_off }); + }, + } + try cg.getValueIfFree(new_temp_index.tracking(cg).short, new_temp_index.toIndex()); + return .{ .index = new_temp_index.toIndex() }; + } + + /// Returns MCV of a limb. + /// Caller does not own return values. + fn toLimbValue(temp: Temp, limb_index: u64, cg: *CodeGen) MCValue { + return temp.tracking(cg).short.toLimbValue(limb_index); + } + + /// Loads `val` to `temp` if `val` is not in regs yet. + /// Returns either MCV of self if copy occurred or `val` if `val` is already in regs. + fn ensureReg(temp: Temp, cg: *CodeGen, val: MCValue) InnerError!MCValue { + if (val.isInRegister()) { + return val; + } else { + const temp_mcv = temp.tracking(cg).short; + try cg.genCopyToReg(.fromByteSize(temp.typeOf(cg).abiSize(cg.pt.zcu)), temp.getReg(cg), val, .{}); + return temp_mcv; + } + } + + /// Loads `temp` to a newly allocated register if it is not in register yet. + fn ensureOffsetable(temp: Temp, cg: *CodeGen) InnerError!struct { Temp, bool } { + const mcv = temp.tracking(cg).short; + if (!mcv.isMemory()) { + return .{ temp, false }; + } else { + const ty = temp.typeOf(cg); + const new_temp = try cg.tempAlloc(ty, .{ .use_frame = false }); + try temp.copy(new_temp, cg, ty); + return .{ new_temp, true }; + } + } + + fn truncateRegister(temp: Temp, cg: *CodeGen) !void { + const temp_tracking = temp.tracking(cg); + const regs = temp_tracking.getRegs(); + assert(regs.len > 0); + return cg.truncateRegister(temp.typeOf(cg), regs[regs.len - 1]); + } + + fn toType(temp: Temp, cg: *CodeGen, ty: Type) void { + return switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => unreachable, + .temp => |temp_index| temp_index.toType(cg, ty), + }; + } + + fn toOffset(temp: Temp, cg: *CodeGen, off: i32) void { + const temp_tracking = cg.inst_tracking.getPtr(temp.index).?; + temp_tracking.short = temp_tracking.short.offset(off); + } +}; + +fn getValue(cg: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { + for (value.getRegs()) |reg| try cg.register_manager.getReg(reg, inst); +} + +fn getValueIfFree(cg: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { + for (value.getRegs()) |reg| if (cg.register_manager.isRegFree(reg)) + try cg.register_manager.getReg(reg, inst); +} + +fn freeValue(cg: *CodeGen, value: MCValue) !void { + switch (value) { + .register => |reg| try cg.freeReg(reg), + inline .register_pair, + .register_triple, + .register_quadruple, + => |regs| for (regs) |reg| try cg.freeReg(reg), + .register_bias, .register_offset => |reg_off| try cg.freeReg(reg_off.reg), + .load_frame => |frame| try cg.freeFrame(frame.index), + else => {}, + } +} + +fn freeReg(cg: *CodeGen, reg: Register) !void { + cg.register_manager.freeReg(reg); +} + +fn freeFrame(cg: *CodeGen, frame: FrameIndex) !void { + tracking_log.debug("free frame {}", .{frame}); + try cg.free_frame_indices.append(cg.gpa, frame); +} + +const MCVAllocOptions = struct { + use_reg: bool = true, + use_frame: bool = true, +}; + +fn canAllocInReg(cg: *CodeGen, ty: Type) bool { + const zcu = cg.pt.zcu; + const abi_size = ty.abiSize(zcu); + const max_abi_size = @as(u32, switch (ty.zigTypeTag(zcu)) { + .float => 16, + else => 8, + }); + return std.math.isPowerOfTwo(abi_size) and abi_size <= max_abi_size; +} + +pub fn allocReg(cg: *CodeGen, ty: Type, inst: ?Air.Inst.Index) !Register { + if (cg.register_manager.tryAllocReg(inst, cg.getAllocatableRegSetForType(ty))) |reg| { + return reg; + } else { + return cg.fail("Cannot allocate register for {} (Zig compiler bug)", .{ty.fmt(cg.pt)}); + } +} + +pub fn allocRegAndLock(cg: *CodeGen, ty: Type) !struct { Register, RegisterManager.RegisterLock } { + const reg = try cg.allocReg(ty, null); + const lock = cg.register_manager.lockRegAssumeUnused(reg); + return .{ reg, lock }; +} + +pub fn allocRegOrMem(cg: *CodeGen, ty: Type, inst: ?Air.Inst.Index, opts: MCVAllocOptions) !MCValue { + const zcu = cg.pt.zcu; + + if (opts.use_reg and cg.canAllocInReg(ty)) { + if (cg.register_manager.tryAllocReg(inst, cg.getAllocatableRegSetForType(ty))) |reg| { + return .{ .register = reg }; + } + } + + if (opts.use_frame) { + const frame_index = try cg.allocFrameIndex(.initType(ty, zcu)); + return .{ .load_frame = .{ .index = frame_index } }; + } + + return cg.fail("Cannot allocate {} with options {} (Zig compiler bug)", .{ ty.fmt(cg.pt), opts }); +} + +fn allocFrameIndex(cg: *CodeGen, alloc: FrameAlloc) !FrameIndex { + const frame_allocs_slice = cg.frame_allocs.slice(); + const frame_size = frame_allocs_slice.items(.abi_size); + const frame_align = frame_allocs_slice.items(.abi_align); + + const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)]; + stack_frame_align.* = stack_frame_align.max(alloc.abi_align); + + for (cg.free_frame_indices.items, 0..) |frame_index, i| { + const abi_size = frame_size[@intFromEnum(frame_index)]; + if (abi_size != alloc.abi_size) continue; + + // reuse frame + const abi_align = &frame_align[@intFromEnum(frame_index)]; + abi_align.* = abi_align.max(alloc.abi_align); + _ = cg.free_frame_indices.swapRemove(i); + tracking_log.debug("reuse frame {}", .{frame_index}); + return frame_index; + } + + const frame_index: FrameIndex = @enumFromInt(cg.frame_allocs.len); + try cg.frame_allocs.append(cg.gpa, alloc); + tracking_log.debug("new frame {}", .{frame_index}); + return frame_index; +} + +fn getAllocatableRegSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { + return abi.getAllocatableRegSet(self.getRegClassForType(ty)); +} + +fn getRegClassForType(self: *CodeGen, ty: Type) Register.Class { + const pt = self.pt; + const zcu = pt.zcu; + + if (ty.isRuntimeFloat()) return .float; + if (ty.isVector(zcu)) unreachable; // TODO implement vector + return .int; +} + +fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type { + const pt = self.pt; + const zcu = pt.zcu; + return self.air.typeOf(inst, &zcu.intern_pool); +} + +fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type { + return Temp.typeOf(.{ .index = inst }, self); +} + +fn floatBits(cg: *CodeGen, ty: Type) ?u16 { + return if (ty.isRuntimeFloat()) ty.floatBits(cg.target) else null; +} + +fn getLimbCount(cg: *CodeGen, ty: Type) u64 { + return std.math.divCeil(u64, ty.abiSize(cg.pt.zcu), 8) catch unreachable; +} + +fn getLimbSize(cg: *CodeGen, ty: Type, limb: u64) bits.Memory.Size { + return .fromByteSize(@min(ty.abiSize(cg.pt.zcu) - limb * 8, 8)); +} + +fn fieldOffset(cg: *CodeGen, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 { + const pt = cg.pt; + const zcu = pt.zcu; + const agg_ty = ptr_agg_ty.childType(zcu); + return switch (agg_ty.containerLayout(zcu)) { + .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)), + .@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) + + (if (zcu.typeToStruct(agg_ty)) |loaded_struct| pt.structPackedFieldBitOffset(loaded_struct, field_index) else 0) - + ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8), + }; +} + +fn reuseTemp( + cg: *CodeGen, + tracking: *InstTracking, + new_inst: Air.Inst.Index, + old_inst: Air.Inst.Index, +) bool { + switch (tracking.short) { + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_bias, + .register_offset, + => for (tracking.short.getRegs()) |tracked_reg| { + if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { + cg.register_manager.registers[tracked_index] = new_inst; + } + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => {}, + } + tracking.reuse(cg, new_inst, old_inst); + return true; +} + +fn resolveCallInfo(cg: *CodeGen, fn_ty: *const InternPool.Key.FuncType) codegen.CodeGenError!abi.CallInfo { + return abi.CCResolver.resolve(cg.pt, cg.gpa, cg.target, fn_ty) catch |err| switch (err) { + error.CCSelectFailed => return cg.fail("Failed to resolve calling convention values", .{}), + else => |e| return e, + }; +} + +inline fn getAirData(cg: *CodeGen, inst: Air.Inst.Index) Air.Inst.Data { + return cg.air.instructions.items(.data)[@intFromEnum(inst)]; +} + +const State = struct { + registers: RegisterManager.TrackedRegisters, + reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, + free_registers: RegisterManager.RegisterBitSet, + next_temp_index: Temp.Index, + inst_tracking_len: u32, + scope_generation: u32, +}; + +fn initRetroactiveState(cg: *CodeGen) State { + var state: State = undefined; + state.next_temp_index = cg.next_temp_index; + state.inst_tracking_len = @intCast(cg.inst_tracking.count()); + state.scope_generation = cg.scope_generation; + cg.scope_generation += 1; + return state; +} + +fn saveRetroactiveState(cg: *CodeGen, state: *State) !void { + const free_registers = cg.register_manager.free_registers; + var it = free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = cg.register_manager.registers[index]; + state.registers[index] = tracked_inst; + state.reg_tracking[index] = cg.inst_tracking.get(tracked_inst).?; + } + state.free_registers = free_registers; +} + +fn saveState(cg: *CodeGen) !State { + var state = cg.initRetroactiveState(); + try cg.saveRetroactiveState(&state); + return state; +} + +fn restoreState(cg: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct { + emit_instructions: bool, + update_tracking: bool, + resurrect: bool, + close_scope: bool, +}) !void { + if (opts.close_scope) { + // kill temps + for (@intFromEnum(state.next_temp_index)..@intFromEnum(cg.next_temp_index)) |temp_i| + try (Temp{ .index = @enumFromInt(temp_i) }).die(cg); + cg.next_temp_index = state.next_temp_index; + + // kill inst results + for ( + cg.inst_tracking.keys()[state.inst_tracking_len..], + cg.inst_tracking.values()[state.inst_tracking_len..], + ) |inst, *tracking| try tracking.die(cg, inst); + cg.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len); + } + + if (opts.resurrect) { + // resurrect temps + for ( + cg.inst_tracking.keys()[0..@intFromEnum(state.next_temp_index)], + cg.inst_tracking.values()[0..@intFromEnum(state.next_temp_index)], + ) |inst, *tracking| try tracking.resurrect(cg, inst, state.scope_generation); + // recurrect inst results + for ( + cg.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len], + cg.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len], + ) |inst, *tracking| try tracking.resurrect(cg, inst, state.scope_generation); + } + for (deaths) |death| try cg.inst_tracking.getPtr(death).?.die(cg, death); + + const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterManager.RegisterLock; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); + + var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterManager.RegisterLock).initCapacity( + stack.get(), + @typeInfo(ExpectedContents).array.len, + ); + defer if (!opts.update_tracking) { + for (reg_locks.items) |lock| cg.register_manager.unlockReg(lock); + reg_locks.deinit(); + }; + + // restore register state + for ( + 0.., + cg.register_manager.registers, + state.registers, + state.reg_tracking, + ) |reg_i, current_slot, target_slot, reg_tracking| { + const reg_index: RegisterManager.TrackedIndex = @intCast(reg_i); + const current_maybe_inst = if (cg.register_manager.isRegIndexFree(reg_index)) null else current_slot; + const target_maybe_inst = if (state.free_registers.isSet(reg_index)) null else target_slot; + + if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst| + assert(cg.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len); + + if (opts.emit_instructions and current_maybe_inst != target_maybe_inst) { + if (current_maybe_inst) |current_inst| + try cg.inst_tracking.getPtr(current_inst).?.spill(cg, current_inst); + if (target_maybe_inst) |target_inst| + try cg.inst_tracking.getPtr(target_inst).?.materialize(cg, target_inst, reg_tracking); + } + if (opts.update_tracking) { + if (current_maybe_inst) |current_inst| { + try cg.inst_tracking.getPtr(current_inst).?.trackSpill(cg, current_inst); + cg.register_manager.freeRegIndex(reg_index); + } + if (target_maybe_inst) |target_inst| { + cg.register_manager.getRegIndexAssumeFree(reg_index, target_inst); + cg.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking); + } + } else if (target_maybe_inst) |_| + try reg_locks.append(cg.register_manager.lockRegIndexAssumeUnused(reg_index)); + } + + // verify register state + if (opts.update_tracking and std.debug.runtime_safety) { + assert(cg.register_manager.free_registers.eql(state.free_registers)); + var used_reg_it = state.free_registers.iterator(.{ .kind = .unset }); + while (used_reg_it.next()) |index| + assert(cg.register_manager.registers[index] == state.registers[index]); + } +} + +const BlockState = struct { + state: State, + relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + + fn deinit(self: *BlockState, gpa: Allocator) void { + self.relocs.deinit(gpa); + self.* = undefined; + } +}; + +/// Generates a AIR block. +fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { + @setEvalBranchQuota(28_600); + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const air_tags = cg.air.instructions.items(.tag); + + for (body) |inst| { + if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue; + + cg_mir_log.debug("{}", .{cg.fmtAir(inst)}); + + cg.reused_operands = .initEmpty(); + try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); + + switch (air_tags[@intFromEnum(inst)]) { + .arg => try cg.airArg(inst), + .ret => try cg.airRet(inst, false), + .ret_safe => try cg.airRet(inst, true), + .ret_load => try cg.airRetLoad(inst), + + .add, .add_wrap => try cg.airArithBinOp(inst, .add), + .sub, .sub_wrap => try cg.airArithBinOp(inst, .sub), + + .bit_and => try cg.airLogicBinOp(inst, .@"and"), + .bit_or => try cg.airLogicBinOp(inst, .@"or"), + .xor => try cg.airLogicBinOp(inst, .xor), + .not => try cg.airNot(inst), + + .bitcast => try cg.airBitCast(inst), + .intcast => try cg.airIntCast(inst), + + .assembly => cg.airAsm(inst) catch |err| switch (err) { + error.AsmParseFail => return error.CodegenFail, + else => |e| return e, + }, + .trap, .breakpoint => try cg.asmInst(.@"break"(0)), + + .ret_addr => try cg.airRetAddr(inst), + .frame_addr => try (try cg.tempInit(.usize, .{ .lea_frame = .{ .index = .stack_frame } })).finish(inst, &.{}, cg), + + .alloc => try cg.airAlloc(inst), + .ret_ptr => try cg.airRetPtr(inst), + .inferred_alloc, .inferred_alloc_comptime => unreachable, + .load => try cg.airLoad(inst), + .store => try cg.airStore(inst, false), + .store_safe => try cg.airStore(inst, true), + + .call => try cg.airCall(inst, .auto), + .call_always_tail => try cg.airCall(inst, .always_tail), + .call_never_tail => try cg.airCall(inst, .never_tail), + .call_never_inline => try cg.airCall(inst, .never_inline), + + .cmp_eq => try cg.airCmp(inst, .{ .cond = .eq, .swap = false, .opti = false }), + .cmp_eq_optimized => try cg.airCmp(inst, .{ .cond = .eq, .swap = false, .opti = true }), + .cmp_neq => try cg.airCmp(inst, .{ .cond = .ne, .swap = false, .opti = false }), + .cmp_neq_optimized => try cg.airCmp(inst, .{ .cond = .ne, .swap = false, .opti = true }), + .cmp_lt => try cg.airCmp(inst, .{ .cond = .gt, .swap = true, .opti = false }), + .cmp_lt_optimized => try cg.airCmp(inst, .{ .cond = .gt, .swap = true, .opti = true }), + .cmp_lte => try cg.airCmp(inst, .{ .cond = .le, .swap = false, .opti = false }), + .cmp_lte_optimized => try cg.airCmp(inst, .{ .cond = .le, .swap = false, .opti = true }), + .cmp_gt => try cg.airCmp(inst, .{ .cond = .gt, .swap = false, .opti = false }), + .cmp_gt_optimized => try cg.airCmp(inst, .{ .cond = .gt, .swap = false, .opti = true }), + .cmp_gte => try cg.airCmp(inst, .{ .cond = .le, .swap = true, .opti = false }), + .cmp_gte_optimized => try cg.airCmp(inst, .{ .cond = .le, .swap = true, .opti = true }), + + .block => try cg.airBlock(inst), + .dbg_inline_block => try cg.airDbgInlineBlock(inst), + .loop => try cg.airLoop(inst), + .br => try cg.airBr(inst), + .repeat => try cg.airRepeat(inst), + .cond_br => try cg.airCondBr(inst), + + .is_err => try cg.airIsErr(inst, false), + .is_non_err => try cg.airIsErr(inst, true), + + .slice_ptr => try cg.airSlicePtr(inst), + .slice_len => try cg.airSliceLen(inst), + .slice_elem_ptr => try cg.airPtrElemPtr(inst), + .ptr_elem_ptr => try cg.airPtrElemPtr(inst), + .slice_elem_val => try cg.airPtrElemVal(inst), + .ptr_elem_val => try cg.airPtrElemVal(inst), + + .struct_field_ptr => try cg.airStructFieldPtr(inst), + .struct_field_ptr_index_0 => try cg.airStructFieldPtrConst(inst, 0), + .struct_field_ptr_index_1 => try cg.airStructFieldPtrConst(inst, 1), + .struct_field_ptr_index_2 => try cg.airStructFieldPtrConst(inst, 2), + .struct_field_ptr_index_3 => try cg.airStructFieldPtrConst(inst, 3), + + .unwrap_errunion_payload => try cg.airUnwrapErrUnionPayload(inst), + .unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst), + .unwrap_errunion_payload_ptr => try cg.airUnwrapErrUnionPayloadPtr(inst), + .unwrap_errunion_err_ptr => try cg.airUnwrapErrUnionErrPtr(inst), + .wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst), + .wrap_errunion_err => try cg.airWrapErrUnionErr(inst), + + .@"try", .try_cold => try cg.airTry(inst), + .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst), + + .unreach => {}, + + .intcast_safe, + .add_safe, + .sub_safe, + .mul_safe, + => return cg.fail("legalization miss (Zig compiler bug)", .{}), + + .dbg_stmt => if (!cg.mod.strip) { + const dbg_stmt = cg.getAirData(inst).dbg_stmt; + try cg.asmPseudo(.dbg_line_stmt_line_column, .{ .line_column = .{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + } }); + }, + .dbg_empty_stmt => if (!cg.mod.strip) { + if (cg.mir_instructions.len > 0) { + const prev_mir_tag = &cg.mir_instructions.items(.tag)[cg.mir_instructions.len - 1]; + if (prev_mir_tag.* == Mir.Inst.Tag.fromPseudo(.dbg_line_line_column)) + prev_mir_tag.* = Mir.Inst.Tag.fromPseudo(.dbg_line_stmt_line_column); + } + try cg.asmInst(.andi(.r0, .r0, 0)); + }, + // TODO: emit debug info + .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => { + const pl_op = cg.getAirData(inst).pl_op; + var ops = try cg.tempsFromOperands(inst, .{pl_op.operand}); + try ops[0].die(cg); + }, + + else => return cg.fail( + "TODO implement {s} for LoongArch64 CodeGen", + .{@tagName(air_tags[@intFromEnum(inst)])}, + ), + } + + try cg.resetTemps(@enumFromInt(0)); + verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); + cg.checkInvariantsAfterAirInst(); + } +} + +const FormatAirData = struct { + self: *CodeGen, + inst: Air.Inst.Index, +}; +fn formatAir( + data: FormatAirData, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) @TypeOf(writer).Error!void { + data.self.air.writeInst(writer, data.inst, data.self.pt, data.self.liveness); +} +fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) { + return .{ .data = .{ .self = self, .inst = inst } }; +} + +const FormatTrackingData = struct { + self: *CodeGen, +}; +fn formatTracking( + data: FormatTrackingData, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) @TypeOf(writer).Error!void { + var it = data.self.inst_tracking.iterator(); + while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* }); + + try writer.writeAll("\nUsed registers:"); + var reg_it = data.self.register_manager.free_registers.iterator(.{ .kind = .unset }); + while (reg_it.next()) |index| try writer.print(" {s}", .{@tagName(RegisterManager.regAtTrackedIndex(@intCast(index)))}); +} +fn fmtTracking(self: *CodeGen) std.fmt.Formatter(formatTracking) { + return .{ .data = .{ .self = self } }; +} + +fn formatFrameLocs( + cg: *CodeGen, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) @TypeOf(writer).Error!void { + for (0..cg.frame_allocs.len) |i| { + try writer.print("\n- {}: {}\n @ {}", .{ @as(FrameIndex, @enumFromInt(i)), cg.frame_allocs.get(i), cg.frame_locs.get(i) }); + } +} +fn fmtFrameLocs(self: *CodeGen) std.fmt.Formatter(formatFrameLocs) { + return .{ .data = self }; +} + +fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void { + if (std.debug.runtime_safety) { + var any_valid = false; + for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + if (temp.isValid(cg)) { + any_valid = true; + log.err("failed to kill {}: {}, tracking: {}", .{ + temp.toIndex(), + cg.temp_type[temp_index].fmt(cg.pt), + temp.tracking(cg), + }); + } + cg.temp_type[temp_index] = undefined; + } + if (any_valid) return cg.fail("failed to kill all temps", .{}); + } + cg.next_temp_index = from_index; +} + +fn tempAlloc(cg: *CodeGen, ty: Type, opts: MCVAllocOptions) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init(try cg.allocRegOrMem(ty, temp_index.toIndex(), opts)); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocReg(cg: *CodeGen, ty: Type, regs: RegisterManager.RegisterBitSet) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), regs) }, + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + .{ .register_pair = try cg.register_manager.allocRegs(2, @splat(temp_index.toIndex()), rs) }, + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocMem(cg: *CodeGen, ty: Type) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + try cg.allocRegOrMem(ty, temp_index.toIndex(), .{ .use_reg = false }), + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempInit(cg: *CodeGen, ty: Type, value: MCValue) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init(value); + cg.temp_type[@intFromEnum(temp_index)] = ty; + try cg.getValue(value, temp_index.toIndex()); + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + tracking_log.debug("{} => {} (birth)", .{ temp_index.toIndex(), value }); + return .{ .index = temp_index.toIndex() }; +} + +fn tempFromValue(cg: *CodeGen, value: Value) InnerError!Temp { + return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerValue(value)); +} + +fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp { + return cg.tempMemFromAlignedValue(.none, value); +} + +fn tempMemFromAlignedValue(cg: *CodeGen, alignment: InternPool.Alignment, value: Value) InnerError!Temp { + const ty = value.typeOf(cg.pt.zcu); + return cg.tempInit(ty, .{ .load_uav = .{ + .val = value.toIntern(), + .orig_ty = (try cg.pt.ptrType(.{ + .child = ty.toIntern(), + .flags = .{ + .is_const = true, + .alignment = alignment, + }, + })).toIntern(), + } }); +} + +fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + + if (op_dies) { + const temp_index = cg.next_temp_index; + const temp: Temp = .{ .index = temp_index.toIndex() }; + const op_inst = op_ref.toIndex().?; + const tracking = cg.resolveInst(op_inst); + temp_index.tracking(cg).* = tracking.*; + tracking_log.debug("{} => {} (birth from operand)", .{ temp_index.toIndex(), tracking.short }); + + if (!cg.reuseTemp(tracking, temp.index, op_inst)) return .{ .index = op_ref.toIndex().? }; + cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return temp; + } + + if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; + const val = op_ref.toInterned().?; + return cg.tempInit(.fromInterned(ip.typeOf(val)), try cg.lowerValue(.fromInterned(val))); +} + +fn tempsFromOperandsInner( + cg: *CodeGen, + inst: Air.Inst.Index, + op_temps: []Temp, + op_refs: []const Air.Inst.Ref, +) InnerError!void { + for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| { + if (op_ref == prev_op_ref) break false; + } else cg.liveness.operandDies(inst, @intCast(op_index))); +} + +inline fn tempsFromOperands( + cg: *CodeGen, + inst: Air.Inst.Index, + op_refs: anytype, +) InnerError![op_refs.len]Temp { + var op_temps: [op_refs.len]Temp = undefined; + try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs); + return op_temps; +} + +fn tempReuseOrAlloc(cg: *CodeGen, inst: Air.Inst.Index, op: Temp, op_i: Air.Liveness.OperandInt, ty: Type, opts: MCVAllocOptions) InnerError!struct { Temp, bool } { + if (cg.liveness.operandDies(inst, op_i) and op.isMut(cg)) { + cg.reused_operands.set(op_i); + return .{ op, true }; + } else { + return .{ try cg.tempAlloc(ty, opts), false }; + } +} + +fn tempTryReuseOrAlloc(cg: *CodeGen, inst: Air.Inst.Index, op_temps: []const Temp, opts: MCVAllocOptions) InnerError!Temp { + for (op_temps, 0..) |op, op_i| { + if (cg.liveness.operandDies(inst, @truncate(op_i)) and op.isMut(cg)) { + cg.reused_operands.set(op_i); + return op; + } + } + return cg.tempAlloc(op_temps[0].typeOf(cg), opts); +} + +fn reuseOperand( + cg: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Index, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, +) bool { + return cg.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); +} + +fn reuseOperandAdvanced( + cg: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Index, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, + maybe_tracked_inst: ?Air.Inst.Index, +) bool { + if (!cg.liveness.operandDies(inst, op_index)) + return false; + + switch (mcv) { + .register, + .register_pair, + .register_triple, + .register_quadruple, + => for (mcv.getRegs()) |reg| { + // If it's in the registers table, need to associate the register(s) with the + // new instruction. + if (maybe_tracked_inst) |tracked_inst| { + if (!cg.register_manager.isRegFree(reg)) { + if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { + cg.register_manager.registers[index] = tracked_inst; + } + } + } else cg.register_manager.freeReg(reg); + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => return false, + } + + // Prevent the operand deaths processing code from deallocating it. + cg.reused_operands.set(op_index); + cg.resolveInst(operand).reuse(cg, inst, operand); + + return true; +} + +fn resolveRef(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue { + const zcu = cg.pt.zcu; + const ty = cg.typeOf(ref); + + // If the type has no codegen bits, no need to store it. + if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none; + + const mcv: MCValue = if (ref.toIndex()) |inst| mcv: { + break :mcv cg.inst_tracking.getPtr(inst).?.short; + } else mcv: { + break :mcv try cg.lowerValue(.fromInterned(ref.toInterned().?)); + }; + + switch (mcv) { + .none, .unreach, .dead => unreachable, + else => return mcv, + } +} + +fn resolveInst(cg: *CodeGen, inst: Air.Inst.Index) *InstTracking { + const tracking = cg.inst_tracking.getPtr(inst).?; + return switch (tracking.short) { + .none, .unreach, .dead => unreachable, + else => tracking, + }; +} + +pub fn spillInstruction(cg: *CodeGen, reg: Register, inst: Air.Inst.Index) !void { + const tracking = cg.inst_tracking.getPtr(inst) orelse return; + if (std.debug.runtime_safety) { + for (tracking.getRegs()) |tracked_reg| { + if (tracked_reg.id() == reg.id()) break; + } else unreachable; // spilled reg not tracked with spilled instruction + } + try tracking.spill(cg, inst); + try tracking.trackSpill(cg, inst); +} + +fn lowerValue(cg: *CodeGen, val: Value) Allocator.Error!MCValue { + return switch (try codegen.lowerValue(cg.pt, val, cg.target)) { + .none => .none, + .undef => .undef, + .immediate => |imm| .{ .immediate = imm }, + .lea_nav => |nav| .{ .lea_nav = .{ .index = nav } }, + .lea_uav => |uav| .{ .lea_uav = .{ .index = uav } }, + .load_uav => |uav| .{ .load_uav = .{ .index = uav } }, + }; +} + +fn checkInvariantsAfterAirInst(cg: *CodeGen) void { + assert(!cg.register_manager.lockedRegsExist()); + + if (std.debug.runtime_safety) { + // check consistency of tracked registers + var it = cg.register_manager.free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = cg.register_manager.registers[index]; + const tracking = cg.resolveInst(tracked_inst); + for (tracking.getRegs()) |reg| { + if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; + } else unreachable; // tracked register not in use + } + } +} + +fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, op: Air.Inst.Ref) !void { + if (bt.feed()) if (op.toIndex()) |inst| try cg.inst_tracking.getPtr(inst).?.die(cg, inst); +} + +fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: MCValue) !void { + if (cg.liveness.isUnused(inst) and cg.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) { + .none, .dead, .unreach => {}, + else => unreachable, // Why didn't the result die? + } else { + tracking_log.debug("{} => {} (birth)", .{ inst, result }); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result)); + try cg.getValueIfFree(result, inst); + } +} + +fn finishAir( + cg: *CodeGen, + inst: Air.Inst.Index, + result: MCValue, + operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, +) !void { + const tomb_bits = cg.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (cg.reused_operands.isSet(op_index)) continue; + try cg.inst_tracking.getPtr(op.toIndexAllowNone() orelse continue).?.die(cg, inst); + } + cg.finishAirResult(inst, result); +} + +const CopyOptions = struct { + safety: bool = false, +}; + +fn genCopy(cg: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) !void { + if (dst_mcv == .none) return; + const zcu = cg.pt.zcu; + if (!ty.hasRuntimeBits(zcu)) return; + + log.debug("copying {} to {} ({}, safety = {})", .{ src_mcv, dst_mcv, ty.fmt(cg.pt), opts.safety }); + switch (dst_mcv) { + .register => |reg| try cg.genCopyToReg(.fromByteSize(ty.abiSize(zcu)), reg, src_mcv, opts), + inline .register_pair, .register_triple, .register_quadruple => |regs| { + for (regs, 0..) |reg, reg_i| { + try cg.genCopyToReg(cg.getLimbSize(ty, reg_i), reg, src_mcv.toLimbValue(reg_i), opts); + } + }, + .load_frame, .load_nav, .load_uav => try cg.genCopyToMem(ty, dst_mcv, src_mcv), + else => return cg.fail("TODO: genCopy {s} => {s}", .{ @tagName(src_mcv), @tagName(dst_mcv) }), + } +} + +fn genCopyToReg(cg: *CodeGen, size: bits.Memory.Size, dst: Register, src_mcv: MCValue, opts: CopyOptions) !void { + if (size == .dword and !cg.hasFeature(.@"64bit")) return cg.fail("Cannot copy double-words to register on LA32", .{}); + switch (src_mcv) { + .none => {}, + .dead, .unreach => unreachable, + .undef => if (opts.safety) try cg.asmInst(.lu12i_w(dst, 0xaaaa)), + .register => |src| if (dst != src) try cg.asmInst(.@"or"(dst, src, .zero)), + .register_bias => |ro| { + const off = cast(i12, ro.off) orelse return cg.fail("TODO copy reg_bias to reg", .{}); + switch (cg.target.cpu.arch) { + .loongarch32 => try cg.asmInst(.addi_w(dst, ro.reg, off)), + .loongarch64 => try cg.asmInst(.addi_d(dst, ro.reg, off)), + else => unreachable, + } + }, + .immediate => |imm| try cg.asmPseudo(.imm_to_reg, .{ .imm_reg = .{ + .imm = imm, + .reg = dst, + } }), + .register_offset => |ro| { + if (cast(i12, ro.off)) |off| { + switch (size) { + .byte => try cg.asmInst(.ld_bu(dst, ro.reg, off)), + .hword => try cg.asmInst(.ld_hu(dst, ro.reg, off)), + .word => try cg.asmInst(.ld_w(dst, ro.reg, off)), + .dword => try cg.asmInst(.ld_d(dst, ro.reg, off)), + } + return; + } + if (cast(i16, ro.off)) |off| { + if (off & 0b11 == 0) { + switch (size) { + .word => return cg.asmInst(.ldox4_w(dst, ro.reg, @intCast(off >> 2))), + .dword => return cg.asmInst(.ldox4_d(dst, ro.reg, @intCast(off >> 2))), + else => {}, + } + } + } + try cg.genCopyToReg(.dword, dst, .{ .register_bias = .{ .reg = ro.reg, .off = ro.off } }, .{}); + return cg.genCopyToReg(size, dst, .{ .register_offset = .{ .reg = dst } }, .{}); + }, + .lea_frame => |addr| try cg.asmPseudo(.frame_addr_to_reg, .{ .frame_reg = .{ + .frame = addr, + .reg = dst, + } }), + .load_frame => |addr| { + try cg.asmPseudo(.frame_addr_reg_mem, .{ .memop_frame_reg = .{ + .op = .{ + .op = .load, + .signedness = .unsigned, + .size = size, + }, + .frame = addr, + .reg = dst, + .tmp_reg = dst, + } }); + }, + .lea_nav => |nav| try cg.asmPseudo(.nav_addr_to_reg, .{ .nav_reg = .{ + .nav = nav, + .reg = dst, + } }), + .load_nav => |nav| { + try cg.asmPseudo(.nav_memop, .{ .memop_nav_reg = .{ + .op = .{ + .op = .load, + .signedness = .unsigned, + .size = size, + }, + .nav = nav, + .reg = dst, + .tmp_reg = dst, + } }); + }, + .lea_uav => |uav| try cg.asmPseudo(.uav_addr_to_reg, .{ .uav_reg = .{ + .uav = uav, + .reg = dst, + } }), + .load_uav => |uav| { + try cg.asmPseudo(.uav_memop, .{ .memop_uav_reg = .{ + .op = .{ + .op = .load, + .signedness = .unsigned, + .size = size, + }, + .uav = uav, + .reg = dst, + .tmp_reg = dst, + } }); + }, + else => return cg.fail("TODO: genCopyToReg from {s}", .{@tagName(src_mcv)}), + } +} + +/// Copies a value from register to memory. +fn genCopyRegToMem(cg: *CodeGen, dst_mcv: MCValue, src: Register, size: bits.Memory.Size) !void { + if (size == .dword and !cg.hasFeature(.@"64bit")) return cg.fail("Cannot copy double-words from register to memory on LA32", .{}); + switch (dst_mcv) { + else => unreachable, + .air_ref => |dst_ref| try cg.genCopyRegToMem(try cg.resolveRef(dst_ref), src, size), + .memory => |addr| { + const tmp_reg, const tmp_reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(tmp_reg_lock); + try cg.genCopyToReg(.dword, tmp_reg, .{ .immediate = addr }, .{}); + try cg.genCopyRegToMem(.{ .register_offset = .{ .reg = tmp_reg } }, src, size); + }, + .register_offset => |ro| { + if (cast(i12, ro.off)) |off| { + switch (size) { + .byte => try cg.asmInst(.st_b(src, ro.reg, off)), + .hword => try cg.asmInst(.st_h(src, ro.reg, off)), + .word => try cg.asmInst(.st_w(src, ro.reg, off)), + .dword => try cg.asmInst(.st_d(src, ro.reg, off)), + } + return; + } + if (cast(i16, ro.off)) |off| { + if (off & 0b11 == 0) { + switch (size) { + .word => return cg.asmInst(.stox4_w(src, ro.reg, @intCast(off >> 2))), + .dword => return cg.asmInst(.stox4_d(src, ro.reg, @intCast(off >> 2))), + else => {}, + } + } + } + return cg.fail("TODO: genCopyRegToMem to {s}", .{@tagName(dst_mcv)}); + }, + .load_frame => |addr| { + const tmp_reg, const tmp_reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(tmp_reg_lock); + try cg.asmPseudo(.frame_addr_reg_mem, .{ .memop_frame_reg = .{ + .op = .{ + .op = .store, + .size = size, + .signedness = undefined, + }, + .frame = addr, + .reg = src, + .tmp_reg = tmp_reg, + } }); + }, + .load_nav => |nav_off| { + const tmp_reg, const tmp_reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(tmp_reg_lock); + try cg.asmPseudo(.nav_memop, .{ .memop_nav_reg = .{ + .op = .{ + .op = .store, + .size = size, + .signedness = undefined, + }, + .nav = nav_off, + .reg = src, + .tmp_reg = tmp_reg, + } }); + }, + .load_uav => |uav_off| { + const tmp_reg, const tmp_reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(tmp_reg_lock); + try cg.asmPseudo(.uav_memop, .{ .memop_uav_reg = .{ + .op = .{ + .op = .store, + .size = size, + .signedness = undefined, + }, + .uav = uav_off, + .reg = src, + .tmp_reg = tmp_reg, + } }); + }, + .undef, + .load_lazy_sym, + => return cg.fail("TODO: genCopyRegToMem to {s}", .{@tagName(dst_mcv)}), + } +} + +fn genCopyToMem(cg: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { + const pt = cg.pt; + const zcu = pt.zcu; + + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + switch (src_mcv) { + .none, + .unreach, + .dead, + .reserved_frame, + => unreachable, + .air_ref => |src_ref| try cg.genCopyToMem(ty, dst_mcv, try cg.resolveRef(src_ref)), + .register => |reg| return cg.genCopyRegToMem(dst_mcv, reg, .fromByteSize(abi_size)), + .memory, + .undef, + => return cg.fail("TODO: genCopyToMem from {s}", .{@tagName(src_mcv)}), + inline .register_pair, .register_triple, .register_quadruple => |regs| { + for (regs, 0..) |reg, reg_i| { + const size: bits.Memory.Size = limb_size: { + if (reg_i != regs.len - 1) break :limb_size .dword else { + const size = abi_size % 8; + break :limb_size if (size == 0) .dword else .fromByteSize(size); + } + }; + try cg.genCopyRegToMem(dst_mcv.toLimbValue(reg_i), reg, size); + } + }, + .immediate, .register_bias, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => { + const tmp_reg, const tmp_reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(tmp_reg_lock); + + try cg.genCopyToReg(.dword, tmp_reg, src_mcv, .{}); + try cg.genCopyRegToMem(dst_mcv, tmp_reg, .dword); + }, + .register_frame => |reg_frame| { + // copy reg + try cg.genCopyRegToMem(dst_mcv, reg_frame.reg, .dword); + // copy memory + return cg.fail("TODO: genCopyToMem from {s}", .{@tagName(src_mcv)}); + }, + .register_offset, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + const reg, const reg_lock = try cg.allocRegAndLock(.usize); + defer cg.register_manager.unlockReg(reg_lock); + for (0..@intCast(cg.getLimbCount(ty))) |limb_i| { + const size = cg.getLimbSize(ty, limb_i); + try cg.genCopyToReg(size, reg, src_mcv.toLimbValue(limb_i), .{}); + try cg.genCopyRegToMem(dst_mcv.toLimbValue(limb_i), reg, size); + } + }, + } +} + +/// Truncates the value in the register in place. +/// Clobbers any remaining bits. +/// 32-bit values will not be truncated. +fn truncateRegister(cg: *CodeGen, ty: Type, reg: Register) !void { + const zcu = cg.pt.zcu; + + assert(reg.class() == .int); + const reg_size = cg.intRegisterSize(); + const bit_size = @as(u6, @intCast(ty.bitSize(zcu) % reg_size)); + + // skip unneeded truncation + if (bit_size == 0 or bit_size == 32) return; + + if (ty.isAbiInt(zcu)) { + if (bit_size <= 32) { + try cg.asmInst(.bstrpick_w(reg, reg, 0, @intCast(bit_size - 1))); + } else { + try cg.asmInst(.bstrpick_d(reg, reg, 0, bit_size - 1)); + } + } else { + if (reg_size == 32) { + try cg.asmInst(.bstrpick_w(reg, reg, 0, @intCast(bit_size - 1))); + } else { + try cg.asmInst(.bstrpick_d(reg, reg, 0, bit_size - 1)); + } + } +} + +/// Pattern matching framework. +const Select = struct { + cg: *CodeGen, + inst: ?Air.Inst.Index, + + ops: [Air.Liveness.bpi]Temp = undefined, + op_types: [Air.Liveness.bpi]Type = undefined, + ops_count: usize = 0, + temps: [8]Temp = undefined, + temps_count: usize = 0, + case_i: usize = 0, + + const Error = InnerError || error{SelectFailed}; + + fn init(cg: *CodeGen, inst: ?Air.Inst.Index, ops: []const Temp) Select { + var sel = Select{ .cg = cg, .inst = inst }; + sel.ops_count = ops.len; + for (ops, 0..) |op, op_i| { + sel.ops[op_i] = op; + sel.op_types[op_i] = op.typeOf(cg); + } + return sel; + } + + fn finish(sel: *Select, result: Temp) !void { + cg_select_log.debug("select finished: {}", .{result}); + if (sel.inst) |inst| { + try result.finish(inst, sel.ops[0..sel.ops_count], sel.cg); + } + for (sel.temps[0..sel.temps_count]) |temp| { + if (temp.index == result.index) continue; + if (Temp.Index.fromIndex(temp.index).isValid(sel.cg)) try temp.die(sel.cg); + } + } + + fn fail(sel: *Select) error{ OutOfMemory, CodegenFail } { + cg_select_log.debug("select failed after {} cases", .{sel.case_i}); + return sel.cg.fail("Failed to select", .{}); + } + + inline fn match(sel: *Select, case: Case) !bool { + sel.case_i += 1; + if (!case.requirement) { + cg_select_log.debug("case {} miss for pre-requirement", .{sel.case_i}); + return false; + } + for (case.required_features.features) |maybe_feature| { + if (maybe_feature) |feature| { + if (!sel.cg.hasFeature(feature)) { + cg_select_log.debug("case {} miss for missing feature: {s}", .{ sel.case_i, @tagName(feature) }); + return false; + } + } + } + + patterns: for (case.patterns, 0..) |pattern, pat_i| { + std.mem.swap(Temp, &sel.ops[pattern.commute[0]], &sel.ops[pattern.commute[1]]); + + for (pattern.srcs, 0..) |src, src_i| { + if (!src.matches(sel.ops[src_i], sel.cg)) { + std.mem.swap(Temp, &sel.ops[pattern.commute[0]], &sel.ops[pattern.commute[1]]); + cg_select_log.debug("case {} pattern {} miss for src {}", .{ sel.case_i, pat_i + 1, src_i + 1 }); + continue :patterns; + } + } + + cg_select_log.debug("case {} pattern {} matched", .{ sel.case_i, pat_i + 1 }); + + for (pattern.srcs, 0..) |src, src_i| { + while (try src.convert(&sel.ops[src_i], sel.cg)) {} + } + break; + } else return false; + + sel.temps_count = case.temps.len; + for (case.temps, 0..) |temp, temp_i| sel.temps[temp_i] = try temp.create(sel); + + return true; + } + + pub const Case = struct { + required_features: FeatureRequirement = .none, + requirement: bool = true, + patterns: []const SrcPattern, + temps: []const TempSpec = &.{}, + }; + + pub const FeatureRequirement = struct { + features: [4]?std.Target.loongarch.Feature = @splat(null), + + const none: FeatureRequirement = .{}; + + fn init(comptime features: packed struct { + la64: bool = false, + f: bool = false, + d: bool = false, + lsx: bool = false, + lasx: bool = false, + }) FeatureRequirement { + var result: FeatureRequirement = .{}; + var result_i: usize = 0; + + if (features.la64) { + result.features[result_i] = .@"64bit"; + result_i += 1; + } + if (features.f) { + result.features[result_i] = .f; + result_i += 1; + } + if (features.d) { + result.features[result_i] = .d; + result_i += 1; + } + if (features.lasx) { + result.features[result_i] = .lasx; + result_i += 1; + } else if (features.lsx) { + result.features[result_i] = .lsx; + result_i += 1; + } + + return result; + } + }; + + pub const SrcPattern = struct { + srcs: []const Src, + commute: struct { u8, u8 } = .{ 0, 0 }, + + pub const Src = union(enum) { + none, + any, + zero, + imm, + imm_val: i32, + imm_fit: u5, + mem, + to_mem, + mut_mem, + to_mut_mem, + reg: Register.Class, + to_reg: Register.Class, + mut_reg: Register.Class, + to_mut_reg: Register.Class, + regs, + reg_frame, + + pub const imm12: Src = .{ .imm_fit = 12 }; + pub const imm20: Src = .{ .imm_fit = 20 }; + /// Immediate 0, see also zero + pub const imm_zero: Src = .{ .imm_val = 0 }; + pub const imm_one: Src = .{ .imm_val = 1 }; + pub const int_reg: Src = .{ .reg = .int }; + pub const to_int_reg: Src = .{ .to_reg = .int }; + pub const int_mut_reg: Src = .{ .mut_reg = .int }; + pub const to_int_mut_reg: Src = .{ .to_mut_reg = .int }; + + fn matches(pat: Src, temp: Temp, cg: *CodeGen) bool { + return switch (pat) { + .none => temp.tracking(cg).short == .none, + .any => true, + .zero => switch (temp.tracking(cg).short) { + .immediate => |imm| imm == 0, + .register => |reg| reg == Register.zero, + else => false, + }, + .imm => temp.tracking(cg).short == .immediate, + .imm_val => |val| switch (temp.tracking(cg).short) { + .immediate => |imm| @as(i32, @intCast(imm)) == val, + else => false, + }, + .imm_fit => |max_size| switch (temp.tracking(cg).short) { + .immediate => |imm| (imm >> max_size) == 0, + else => false, + }, + .mem => temp.tracking(cg).short.isMemory(), + .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(), + .to_mem, .to_mut_mem => true, + .reg => |rc| temp.tracking(cg).short.isRegisterOf(rc), + .to_reg => |_| temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8, + .mut_reg => |rc| temp.isMut(cg) and temp.tracking(cg).short.isRegisterOf(rc), + .to_mut_reg => |_| temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8, + .regs => temp.tracking(cg).short.isInRegister(), + .reg_frame => temp.tracking(cg).short == .register_frame, + }; + } + + fn convert(pat: Src, temp: *Temp, cg: *CodeGen) InnerError!bool { + return switch (pat) { + .none, .any, .zero, .imm, .imm_val, .imm_fit, .regs, .reg_frame => false, + .mem, .to_mem => try temp.moveToMemory(cg, false), + .mut_mem, .to_mut_mem => try temp.moveToMemory(cg, true), + .reg, .to_reg => |rc| try temp.moveToRegister(cg, rc, false), + .mut_reg, .to_mut_reg => |rc| try temp.moveToRegister(cg, rc, true), + }; + } + }; + }; + + pub const TempSpec = struct { + type: Type = .noreturn, + kind: Kind = .any, + + pub const Kind = union(enum) { + alloc: MCVAllocOptions, + mcv: MCValue, + constant: Value, + lazy_symbol: struct { kind: link.File.LazySymbol.Kind }, + + pub const none: Kind = .{ .mcv = .none }; + pub const undef: Kind = .{ .mcv = .undef }; + pub const any: Kind = .{ .alloc = .{} }; + pub const any_reg: Kind = .{ .alloc = .{ .use_frame = false } }; + pub const any_frame: Kind = .{ .alloc = .{ .use_reg = false } }; + }; + + pub const any_usize_reg: TempSpec = .{ .type = .usize, .kind = .any_reg }; + + fn create(spec: TempSpec, sel: *const Select) InnerError!Temp { + const cg = sel.cg; + const pt = cg.pt; + const zcu = pt.zcu; + + return switch (spec.kind) { + .alloc => |alloc_opts| try cg.tempAlloc(spec.type, alloc_opts), + .mcv => |mcv| try cg.tempInit(spec.type, mcv), + .constant => |constant| try cg.tempInit(constant.typeOf(zcu), try cg.lowerValue(constant)), + .lazy_symbol => |lazy_symbol_spec| { + const ip = &pt.zcu.intern_pool; + const ty = spec.type; + const lazy_symbol: link.File.LazySymbol = .{ + .kind = lazy_symbol_spec.kind, + .ty = switch (ip.indexToKey(ty.toIntern())) { + .inferred_error_set_type => |func_index| switch (ip.funcIesResolvedUnordered(func_index)) { + .none => unreachable, // unresolved inferred error set + else => |ty_index| ty_index, + }, + else => ty.toIntern(), + }, + }; + return try cg.tempInit(.usize, .{ .lea_lazy_sym = lazy_symbol }); + }, + }; + } + }; +}; + +fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void { + var arg_index = cg.arg_index; + while (cg.args_mcv[arg_index] == .none) arg_index += 1; + const arg_mcv = cg.args_mcv[arg_index]; + cg.arg_index = arg_index + 1; + + try cg.finishAirResult(inst, arg_mcv); +} + +fn airRet(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { + const pt = cg.pt; + const zcu = pt.zcu; + const un_op = cg.getAirData(inst).un_op; + + const op_temp = try cg.tempFromOperand(un_op, false); + + // try to skip the copy + if (op_temp.maybeTracking(cg)) |op_tracking| { + if (std.meta.eql(op_tracking.short, cg.ret_mcv)) { + try op_temp.finish(inst, &.{op_temp}, cg); + try cg.finishReturn(inst); + return; + } + } + + // copy return values into proper location + const ret_ty = cg.fn_type.fnReturnType(zcu); + const ret_temp = switch (cg.call_info.return_value) { + .ref_frame => |_| deref_ret: { + // load pointer + const ret_ptr = try cg.tempInit(.usize, cg.ret_mcv); + const ret_temp = try ret_ptr.load(cg, ret_ty, .{ .safety = safety }); + try ret_ptr.die(cg); + break :deref_ret ret_temp; + }, + else => try cg.tempInit(ret_ty, cg.ret_mcv), + }; + try op_temp.copy(ret_temp, cg, ret_ty); + + try ret_temp.finish(inst, &.{op_temp}, cg); + try cg.finishReturn(inst); +} + +fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void { + const pt = cg.pt; + const zcu = pt.zcu; + const un_op = cg.getAirData(inst).un_op; + + var un_temp = try cg.tempFromOperand(un_op, false); + switch (cg.call_info.return_value) { + // per AIR semantics, when the return value is passed by-ref, operand is always ret_ptr. + .ref_register, .ref_frame => {}, + // or, load from memory + else => { + const ret_ty = cg.fn_type.fnReturnType(zcu); + const ret_temp = try cg.tempInit(ret_ty, cg.ret_mcv); + try un_temp.loadTo(ret_temp, cg, ret_ty, .{}); + try ret_temp.die(cg); + }, + } + + try (try cg.tempInit(.noreturn, .unreach)).finish(inst, &.{un_temp}, cg); + try cg.finishReturn(inst); +} + +/// Finishes a return. +/// The return value is expected to be copied to the proper location. +fn finishReturn(cg: *CodeGen, inst: Air.Inst.Index) !void { + _ = inst; + + // restore error return trace + if (cg.call_info.err_ret_trace_reg) |err_ret_trace_reg| { + if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { + if (switch (err_ret_trace.short) { + .register => |reg| err_ret_trace_reg != reg, + else => true, + }) try cg.genCopy(.usize, .{ .register = err_ret_trace_reg }, err_ret_trace.short, .{}); + err_ret_trace.liveOut(cg, err_ret_trace_index); + } + } + + // jump to epilogue + try cg.asmPseudo(.jump_to_epilogue, .none); +} + +fn airArithBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) !void { + const pt = cg.pt; + const zcu = pt.zcu; + + const bin_op = cg.getAirData(inst).bin_op; + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs })); + const ty = sel.ops[0].typeOf(cg); + + // case 1: 32-bit integers + if (try sel.match(.{ + .requirement = ty.isInt(zcu) and + ty.intInfo(zcu).bits <= 32, + .patterns = &.{.{ .srcs = &.{ .to_int_reg, .to_int_reg } }}, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{ lhs, rhs }, .{ .use_frame = false }); + switch (op) { + .add => try cg.asmInst(.add_w(dst.getReg(cg), lhs.getReg(cg), rhs.getReg(cg))), + .sub => try cg.asmInst(.sub_w(dst.getReg(cg), lhs.getReg(cg), rhs.getReg(cg))), + } + try dst.truncateRegister(cg); + try sel.finish(dst); + } else + // case 2: 64-bit integers + if (try sel.match(.{ + .required_features = .init(.{ .la64 = true }), + .requirement = ty.isInt(zcu) and + ty.intInfo(zcu).bits <= 64, + .patterns = &.{.{ .srcs = &.{ .to_int_reg, .to_int_reg } }}, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{ lhs, rhs }, .{ .use_frame = false }); + switch (op) { + .add => try cg.asmInst(.add_d(dst.getReg(cg), lhs.getReg(cg), rhs.getReg(cg))), + .sub => try cg.asmInst(.sub_d(dst.getReg(cg), lhs.getReg(cg), rhs.getReg(cg))), + } + try dst.truncateRegister(cg); + try sel.finish(dst); + } else return sel.fail(); +} + +const LogicBinOpKind = enum { @"and", @"or", xor }; + +fn asmIntLogicBinOpRRR(cg: *CodeGen, op: LogicBinOpKind, dst: Register, src1: Register, src2: Register) !void { + return switch (op) { + .@"and" => cg.asmInst(.@"and"(dst, src1, src2)), + .@"or" => cg.asmInst(.@"or"(dst, src1, src2)), + .xor => cg.asmInst(.xor(dst, src1, src2)), + }; +} + +fn asmIntLogicBinOpRRI(cg: *CodeGen, op: LogicBinOpKind, dst: Register, src1: Register, src2: u12) !void { + return switch (op) { + .@"and" => cg.asmInst(.andi(dst, src1, src2)), + .@"or" => cg.asmInst(.ori(dst, src1, src2)), + .xor => cg.asmInst(.xori(dst, src1, src2)), + }; +} + +fn airLogicBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: LogicBinOpKind) !void { + const pt = cg.pt; + const zcu = pt.zcu; + + const bin_op = cg.getAirData(inst).bin_op; + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs })); + const ty = sel.ops[0].typeOf(cg); + assert(ty.isAbiInt(zcu)); + + // case 1: RI + if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .imm12 } }, + .{ .srcs = &.{ .to_int_reg, .imm12 }, .commute = .{ 0, 1 } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst, _ = try cg.tempReuseOrAlloc(inst, lhs, 0, ty, .{ .use_frame = false }); + const lhs_limb = lhs.toLimbValue(0, cg); + const dst_limb = dst.toLimbValue(0, cg); + try asmIntLogicBinOpRRI(cg, op, dst_limb.getReg().?, lhs_limb.getReg().?, @intCast(rhs.getUnsignedImm(cg))); + try sel.finish(dst); + } + // case 2: RR + else if (try sel.match(.{ + .patterns = &.{.{ .srcs = &.{ .to_int_reg, .to_int_reg } }}, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst, _ = try cg.tempReuseOrAlloc(inst, lhs, 0, ty, .{ .use_frame = false }); + for (0..@intCast(lhs.getLimbCount(cg))) |limb_i| { + const lhs_limb = lhs.toLimbValue(limb_i, cg); + const rhs_limb = rhs.toLimbValue(limb_i, cg); + const dst_limb = dst.toLimbValue(limb_i, cg); + try asmIntLogicBinOpRRR(cg, op, dst_limb.getReg().?, lhs_limb.getReg().?, rhs_limb.getReg().?); + } + try sel.finish(dst); + } + // case 3: limbs + else if (try sel.match(.{ + .patterns = &.{.{ .srcs = &.{ .any, .any } }}, + .temps = &.{ .any_usize_reg, .any_usize_reg }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const tmp1, const tmp2 = sel.temps[0..2].*; + const dst, _ = try cg.tempReuseOrAlloc(inst, lhs, 0, ty, .{ .use_frame = false }); + for (0..@intCast(lhs.getLimbCount(cg))) |limb_i| { + const lhs_limb = try tmp1.ensureReg(cg, lhs.toLimbValue(limb_i, cg)); + const rhs_limb = try tmp2.ensureReg(cg, rhs.toLimbValue(limb_i, cg)); + const dst_limb = dst.toLimbValue(limb_i, cg); + try asmIntLogicBinOpRRR(cg, op, dst_limb.getReg().?, lhs_limb.getReg().?, rhs_limb.getReg().?); + } + try sel.finish(dst); + } else return sel.fail(); +} + +fn airNot(cg: *CodeGen, inst: Air.Inst.Index) !void { + const pt = cg.pt; + const zcu = pt.zcu; + + const ty_op = cg.getAirData(inst).ty_op; + const ty = ty_op.ty.toType(); + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ty_op.operand})); + + // case 1: booleans + if (try sel.match(.{ + .requirement = ty.zigTypeTag(zcu) == .bool, + .patterns = &.{ + .{ .srcs = &.{.to_int_reg} }, + }, + })) { + const op = sel.ops[0]; + const dst, _ = try cg.tempReuseOrAlloc(inst, op, 0, ty, .{ .use_frame = false }); + try cg.asmInst(.xori(dst.getReg(cg), op.getReg(cg), 1)); + try sel.finish(dst); + } + // case 2: integers, fit in one register + else if (try sel.match(.{ + .requirement = ty.isInt(zcu), + .patterns = &.{.{ .srcs = &.{.to_int_reg} }}, + })) { + const op = sel.ops[0]; + const dst, _ = try cg.tempReuseOrAlloc(inst, op, 0, ty, .{ .use_frame = false }); + try cg.asmInst(.nor(dst.getReg(cg), op.getReg(cg), .zero)); + try sel.finish(dst); + } + // case 3: integers, per-limb + else if (try sel.match(.{ + .requirement = ty.isInt(zcu), + .patterns = &.{.{ .srcs = &.{.any} }}, + .temps = &.{.any_usize_reg}, + })) { + const op = sel.ops[0]; + const tmp = sel.temps[0]; + const dst, _ = try cg.tempReuseOrAlloc(inst, op, 0, ty, .{ .use_frame = false }); + for (0..@intCast(op.getLimbCount(cg))) |limb_i| { + const op_limb = try tmp.ensureReg(cg, op.toLimbValue(limb_i, cg)); + const dst_limb = dst.toLimbValue(limb_i, cg); + try cg.asmInst(.nor(dst_limb.getReg().?, op_limb.getReg().?, .zero)); + } + try sel.finish(dst); + } else return sel.fail(); +} + +fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) !void { + // do not mark $ra as allocated + const index = RegisterManager.indexOfKnownRegIntoTracked(.ra).?; + const ra_allocated = cg.register_manager.allocated_registers.isSet(index); + const dst = try cg.tempInit(.usize, .{ .register = .ra }); + cg.register_manager.allocated_registers.setValue(index, ra_allocated); + + try cg.asmPseudo(.load_ra, .none); + try dst.finish(inst, &.{}, cg); +} + +fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty = cg.getAirData(inst).ty; + const child_ty = ty.childType(zcu); + const frame = try cg.allocFrameIndex(.initType(child_ty, zcu)); + const result = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame } }); + try result.finish(inst, &.{}, cg); +} + +fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + switch (cg.call_info.return_value) { + .ref_register => |reg| { + const result = try cg.tempInit(.ptr_usize, .{ .register = reg }); + try result.finish(inst, &.{}, cg); + }, + .ref_frame => |frame_off| { + const result = try cg.tempInit(.ptr_usize, .{ .load_frame = .{ + .index = .args_frame, + .off = frame_off, + } }); + try result.finish(inst, &.{}, cg); + }, + else => try cg.airAlloc(inst), + } +} + +fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const ty = ty_op.ty.toType(); + const val_mcv = try cg.resolveRef(ty_op.operand); + var val = (try cg.tempsFromOperands(inst, .{ty_op.operand}))[0]; + return switch (val_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_pair, + .register_triple, + .register_quadruple, + .register_frame, + .reserved_frame, + .air_ref, + => unreachable, + .memory, .register_offset, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + if (ty.abiSize(zcu) == Type.usize.abiSize(zcu)) { + var tmp = try cg.tempAlloc(.usize, .{ .use_frame = false }); + try val.copy(tmp, cg, .usize); + try tmp.loadTo(tmp, cg, ty, .{}); + tmp.toType(cg, ty); + try tmp.finish(inst, &.{val}, cg); + } else { + const tmp = try cg.tempAlloc(.usize, .{ .use_frame = false }); + try val.copy(tmp, cg, .usize); + const dst = try tmp.load(cg, ty, .{}); + try tmp.die(cg); + try dst.finish(inst, &.{val}, cg); + } + }, + .immediate, .register, .register_bias, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => { + const tmp = try cg.tempAlloc(ty, .{}); + try val.loadTo(tmp, cg, ty, .{}); + try tmp.finish(inst, &.{val}, cg); + }, + }; +} + +fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { + const bin_op = cg.getAirData(inst).bin_op; + var ptr, const val = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + const val_ty = val.typeOf(cg); + + try val.storeTo(ptr, cg, val_ty, .{ .safety = safety }); + + try ptr.die(cg); + try val.die(cg); +} + +fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { + // TODO: tail call + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const gpa = cg.gpa; + const pl_op = cg.getAirData(inst).pl_op; + const callee = pl_op.operand; + const callee_ty = cg.typeOf(callee); + + const fn_ty = zcu.typeToFunc(switch (callee_ty.zigTypeTag(zcu)) { + .@"fn" => callee_ty, + .pointer => callee_ty.childType(zcu), + else => unreachable, + }).?; + var call_info = try cg.resolveCallInfo(&fn_ty); + defer call_info.deinit(gpa); + const ret_ty: Type = .fromInterned(fn_ty.return_type); + + if (modifier == .always_tail) return cg.fail("TODO implement tail calls for loongarch64", .{}); + + const extra = cg.air.extraData(Air.Call, pl_op.payload); + const arg_refs: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]); + + if (arg_refs.len != fn_ty.param_types.len) + return cg.fail("var-arg calls are not supported in LA yet", .{}); + + // adjust call frame size & alignment for the call + cg.adjustFrame(.call_frame, call_info.frame_size, call_info.frame_align); + + var reg_locks = std.ArrayList(RegisterManager.RegisterLock).init(gpa); + defer reg_locks.deinit(); + try reg_locks.ensureTotalCapacity(abi.zigcc.Integer.function_arg_regs.len); + defer for (reg_locks.items) |reg_lock| cg.register_manager.unlockReg(reg_lock); + + // frame indices for by-ref CCVs + const frame_indices = try gpa.alloc(FrameIndex, arg_refs.len); + defer gpa.free(frame_indices); + + // lock regs & alloc frames + for (call_info.params, arg_refs, frame_indices) |ccv, arg_ref, *frame_index| { + // lock regs + for (ccv.getRegs()) |reg| { + try cg.register_manager.getReg(reg, null); + try reg_locks.append(cg.register_manager.lockReg(reg) orelse unreachable); + } + + // alloc frames + switch (ccv) { + .ref_register, .ref_frame => frame_index.* = try cg.allocFrameIndex(.initType(cg.typeOf(arg_ref), zcu)), + else => {}, + } + } + + // lock ret regs + for (call_info.return_value.getRegs()) |reg| { + try cg.register_manager.getReg(reg, null); + if (cg.register_manager.lockReg(reg)) |lock| try reg_locks.append(lock); + } + + // lock temporary regs + for (abi.zigcc.all_temporary) |reg| { + try cg.register_manager.getReg(reg, null); + if (cg.register_manager.lockReg(reg)) |lock| try reg_locks.append(lock); + } + + // resolve ret MCV + const ret_mcv: MCValue = if (cg.liveness.isUnused(inst)) .unreach else ret_mcv: switch (call_info.return_value) { + .ref_register, .ref_frame => { + const frame = try cg.allocFrameIndex(.initType(ret_ty, zcu)); + break :ret_mcv .{ .load_frame = .{ .index = frame } }; + }, + .split => unreachable, + else => |ccv| .fromCCValue(ccv, .call_frame), + }; + + // set arguments in place + for (call_info.params, arg_refs, frame_indices) |ccv, arg_ref, frame_index| { + switch (ccv) { + .ref_register, .ref_frame => { + const dst: MCValue = .{ .load_frame = .{ .index = frame_index } }; + try cg.genCopy(cg.typeOf(arg_ref), dst, try cg.resolveRef(arg_ref), .{}); + }, + else => { + try cg.genCopy(cg.typeOf(arg_ref), .fromCCValue(ccv, .call_frame), try cg.resolveRef(arg_ref), .{}); + }, + } + } + + // lock ra + try cg.register_manager.getReg(.ra, null); + try reg_locks.append(cg.register_manager.lockReg(.ra) orelse unreachable); + + // do the transfer + if (try cg.air.value(pl_op.operand, pt)) |func_value| { + const func_key = ip.indexToKey(func_value.ip_index); + switch (switch (func_key) { + else => func_key, + .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { + .nav => |nav| ip.indexToKey(zcu.navValue(nav).toIntern()), + else => func_key, + } else func_key, + }) { + .func => |func| try cg.asmPseudo(.call, .{ .nav = .{ .index = func.owner_nav } }), + .@"extern" => |ext| try cg.asmPseudo(.call, .{ .nav = .{ .index = ext.owner_nav } }), + // TODO what's this + else => return cg.fail("TODO implement calling bitcasted functions", .{}), + } + } else { + assert(cg.typeOf(callee).zigTypeTag(zcu) == .pointer); + const addr_mcv = try cg.resolveRef(pl_op.operand); + call: { + switch (addr_mcv) { + .register => |reg| break :call try cg.asmInst(.jirl(.ra, reg, 0)), + .register_bias => |ro| if (cast(i18, ro.off)) |off18| + if (off18 & 0b11 == 0) + break :call try cg.asmInst(.jirl(.ra, ro.reg, @truncate(off18 >> 2))), + else => {}, + } + try cg.genCopyToReg(.dword, .t0, addr_mcv, .{}); + try cg.asmInst(.jirl(.ra, .t0, 0)); + } + } + + // finish + var bt = cg.liveness.iterateBigTomb(inst); + try cg.feed(&bt, pl_op.operand); + for (arg_refs) |arg_ref| try cg.feed(&bt, arg_ref); + + try (try cg.tempInit(ret_ty, ret_mcv)).finish(inst, &.{}, cg); +} + +fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const extra = cg.air.extraData(Air.Asm, ty_pl.payload); + var extra_i: usize = extra.end; + const outputs: []const Air.Inst.Ref = + @ptrCast(cg.air.extra.items[extra_i..][0..extra.data.outputs_len]); + extra_i += outputs.len; + const inputs: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra_i..][0..extra.data.inputs_len]); + extra_i += inputs.len; + const clobbers_len = @as(u31, @truncate(extra.data.flags)); + + var parser: AsmParser = .{ + .pt = cg.pt, + .target = cg.target, + .gpa = cg.gpa, + .register_manager = &cg.register_manager, + .src_loc = cg.src_loc, + .output_len = outputs.len, + .input_len = inputs.len, + .clobber_len = clobbers_len, + .mir_offset = cg.label(), + }; + try parser.init(); + defer parser.deinit(); + errdefer if (parser.err_msg) |msg| { + cg.failMsg(msg) catch {}; + }; + + // parse constraints + for (outputs) |_| { + const extra_bytes = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + try parser.parseOutputConstraint(name, constraint); + } + + for (inputs) |_| { + const input_bytes = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(input_bytes, 0); + const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + try parser.parseOutputConstraint(name, constraint); + } + + for (0..clobbers_len) |_| { + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]), 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += clobber.len / 4 + 1; + + try parser.parseClobberConstraint(clobber); + } + + // prepare to input load + try parser.finalizeConstraints(); + + // load inputs + for (inputs, 0..) |input, input_i| { + const input_mcv = &parser.args.items[outputs.len + input_i].value; + const input_cgv = try cg.resolveRef(input); + + switch (input_mcv.*) { + .none => unreachable, + .imm => |*imm| { + switch (input_cgv) { + .immediate => |input_imm| { + if (cast(i32, @as(i64, @bitCast(input_imm)))) |imm32| + imm.* = imm32 + else + return cg.fail("input immediate {} cannot fit into operand", .{imm}); + }, + else => return cg.fail("input {} is not an immediate", .{input_i + 1}), + } + }, + .reg => |reg| { + try cg.genCopyToReg(.fromByteSize(cg.typeOf(input).abiSize(cg.pt.zcu)), reg, input_cgv, .{}); + }, + } + } + + // parse source + const asm_source = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..])[0..extra.data.source_len]; + try parser.parseSource(asm_source); + + // finish MC generation + try parser.finalizeCodeGen(); + + // copy instructions + try cg.mir_instructions.ensureUnusedCapacity(cg.gpa, parser.mir_insts.items.len); + for (parser.mir_insts.items) |mc_inst| + cg.mir_instructions.appendAssumeCapacity(mc_inst); + + // kill operands + var bt = cg.liveness.iterateBigTomb(inst); + for (outputs) |output| if (output != .none) try cg.feed(&bt, output); + for (inputs) |input| try cg.feed(&bt, input); + + // finish assembly block + if (outputs.len != 0) { + assert(outputs.len == 1); + const result_mcv = parser.args.items[0].value; + switch (result_mcv) { + .none, .imm => unreachable, + .reg => |reg| { + const result_ty = cg.typeOfIndex(inst); + const result_tmp = try cg.tempInit(result_ty, .{ .register = reg }); + try result_tmp.finish(inst, &.{}, cg); + }, + } + } else { + try (try cg.tempInit(.void, .none)).finish(inst, &.{}, cg); + } +} + +const CmpOptions = struct { + cond: Mir.BranchCondition.Tag, + swap: bool, + opti: bool, +}; + +// TODO: instruction combination +fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, comptime opts: CmpOptions) !void { + const zcu = cg.pt.zcu; + const bin_op = cg.getAirData(inst).bin_op; + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs })); + if (opts.swap) std.mem.swap(Temp, &sel.ops[0], &sel.ops[1]); + + const ty = sel.ops[0].typeOf(cg); + const cond: Mir.BranchCondition.Tag = + if (ty.isSignedInt(zcu)) + opts.cond + else switch (opts.cond) { + .none, .eq, .ne, .leu, .gtu => opts.cond, + .le => .leu, + .gt => .gtu, + }; + + // case 1: SLTI + if (try sel.match(.{ + .requirement = cond == .gt, + .patterns = &.{ + .{ .srcs = &.{ .imm12, .to_int_reg } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{rhs}, .{ .use_frame = false }); + try cg.asmInst(.slti(dst.getReg(cg), rhs.getReg(cg), @intCast(lhs.getSignedImm(cg)))); + try sel.finish(dst); + } + // case 2: SLTUI + else if (try sel.match(.{ + .requirement = cond == .gtu, + .patterns = &.{ + .{ .srcs = &.{ .imm12, .to_int_reg } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{rhs}, .{ .use_frame = false }); + try cg.asmInst(.sltui(dst.getReg(cg), rhs.getReg(cg), @intCast(lhs.getSignedImm(cg)))); + try sel.finish(dst); + } + // case 3: SLT + else if (try sel.match(.{ + .requirement = cond == .gt, + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .to_int_reg } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{ lhs, rhs }, .{ .use_frame = false }); + try cg.asmInst(.slt(dst.getReg(cg), rhs.getReg(cg), lhs.getReg(cg))); + try sel.finish(dst); + } + // case 4: SLTU + else if (try sel.match(.{ + .requirement = cond == .gtu, + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .to_int_reg } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{ lhs, rhs }, .{ .use_frame = false }); + try cg.asmInst(.sltu(dst.getReg(cg), rhs.getReg(cg), lhs.getReg(cg))); + try sel.finish(dst); + } + // case 5: fallback to conditional branch + else if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .to_int_reg } }, + }, + })) { + const lhs, const rhs = sel.ops[0..2].*; + const dst = try cg.tempTryReuseOrAlloc(inst, &.{ lhs, rhs }, .{ .use_frame = false }); + + const label_if = try cg.asmBr(null, .compare(cond, lhs.getReg(cg), rhs.getReg(cg))); + try cg.asmInst(.ori(dst.getReg(cg), .zero, 0)); + const label_fall = try cg.asmBr(null, .none); + cg.performReloc(label_if); + try cg.asmInst(.ori(dst.getReg(cg), .zero, 1)); + cg.performReloc(label_fall); + + try sel.finish(dst); + } else return sel.fail(); +} + +fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const extra = cg.air.extraData(Air.Block, ty_pl.payload); + + if (!cg.mod.strip) try cg.asmPseudo(.dbg_enter_block, .none); + try cg.lowerBlock(inst, ty_pl.ty, @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); + if (!cg.mod.strip) try cg.asmPseudo(.dbg_exit_block, .none); +} + +fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + + const old_inline_func = cg.inline_func; + defer cg.inline_func = old_inline_func; + cg.inline_func = extra.data.func; + + if (!cg.mod.strip) try cg.asmPseudo(.dbg_enter_inline_func, .{ .func = extra.data.func }); + try cg.lowerBlock(inst, ty_pl.ty, @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); + if (!cg.mod.strip) try cg.asmPseudo(.dbg_enter_inline_func, .{ .func = old_inline_func }); +} + +fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const block = cg.air.extraData(Air.Block, ty_pl.payload); + + try cg.loops.putNoClobber(cg.gpa, inst, .{ + .state = try cg.saveState(), + .target = cg.label(), + }); + defer assert(cg.loops.remove(inst)); + try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); +} + +fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, ty_ref: Air.Inst.Ref, body: []const Air.Inst.Index) !void { + const ty = ty_ref.toType(); + const inst_tracking_i = cg.inst_tracking.count(); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach)); + + // init block data + try cg.blocks.putNoClobber(cg.gpa, inst, .{ .state = cg.initRetroactiveState() }); + const liveness = cg.liveness.getBlock(inst); + + // generate the body + try cg.genBody(body); + + // remove block data + var block_data = cg.blocks.fetchRemove(inst).?.value; + defer block_data.deinit(cg.gpa); + + // if there are any br-s targeting this block + if (block_data.relocs.items.len > 0) { + assert(!ty.eql(.noreturn, cg.pt.zcu)); + try cg.restoreState(block_data.state, liveness.deaths, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + for (block_data.relocs.items) |reloc| cg.performReloc(reloc); + } else assert(ty.eql(.noreturn, cg.pt.zcu)); + + // process return value + if (std.debug.runtime_safety) assert(cg.inst_tracking.getIndex(inst).? == inst_tracking_i); + const tracking = &cg.inst_tracking.values()[inst_tracking_i]; + if (cg.liveness.isUnused(inst)) { + try tracking.die(cg, inst); + } + try cg.getValueIfFree(tracking.short, inst); +} + +fn airBr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const br = cg.getAirData(inst).br; + + const block_ty = cg.typeOfIndex(br.block_inst); + const block_unused = + !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or cg.liveness.isUnused(br.block_inst); + + const block_tracking = cg.inst_tracking.getPtr(br.block_inst).?; + const block_data = cg.blocks.getPtr(br.block_inst).?; + const first_br = block_data.relocs.items.len == 0; + + // prepare/copy the result + const block_result = result: { + if (block_unused) break :result .none; + + if (!first_br) try cg.getValue(block_tracking.short, null); + const op_mcv = try cg.resolveRef(br.operand); + + // try to reuse operand + if (br.operand.toIndex()) |op_index| { + if (cg.reuseOperandAdvanced(inst, op_index, 0, op_mcv, br.block_inst)) { + if (first_br) break :result op_mcv; + + // load value to destination + try cg.getValue(block_tracking.short, br.block_inst); + // .long = .none to avoid merging operand and block result stack frames. + const current_tracking: InstTracking = .{ .long = .none, .short = op_mcv }; + try current_tracking.materializeUnsafe(cg, br.block_inst, block_tracking.*); + try cg.freeValue(op_mcv); + + break :result block_tracking.short; + } + } + + // allocate and copy + const dst_mcv = dst: { + if (first_br) { + break :dst try cg.allocRegOrMem(cg.typeOfIndex(br.block_inst), br.block_inst, .{}); + } else { + try cg.getValue(block_tracking.short, br.block_inst); + break :dst block_tracking.short; + } + }; + try cg.genCopy(block_ty, dst_mcv, op_mcv, .{}); + break :result dst_mcv; + }; + + // process operand death + if (cg.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try cg.inst_tracking.getPtr(op_inst).?.die(cg, op_inst); + } + + if (first_br) { + block_tracking.* = .init(block_result); + try cg.saveRetroactiveState(&block_data.state); + } else { + try cg.restoreState(block_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + } + + // jump to block ends + const jmp_reloc = try cg.asmBr(null, .none); + try block_data.relocs.append(cg.gpa, jmp_reloc); + + // stop tracking block result without forgetting tracking info + try cg.freeValue(block_tracking.short); +} + +fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) !void { + const repeat = cg.getAirData(inst).repeat; + const loop = cg.loops.get(repeat.loop_inst).?; + + try cg.restoreState(loop.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + _ = try cg.asmBr(loop.target, .none); +} + +fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const pt = cg.pt; + const zcu = pt.zcu; + + const pl_op = cg.getAirData(inst).pl_op; + const extra = cg.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_cond_br = cg.liveness.getCondBr(inst); + + const cond_mcv = try cg.resolveRef(pl_op.operand); + const cond_ty = cg.typeOf(pl_op.operand); + + // try to kill the operand earlier so it does not need to be spilled + if (cg.liveness.operandDies(inst, 0)) + if (pl_op.operand.toIndex()) |index| try cg.resolveInst(index).die(cg, index); + + const state = try cg.saveState(); + + // do the branch + const reloc = gen_br: switch (cond_mcv) { + .register => |reg| try cg.asmBr(null, .{ .eq = .{ reg, .zero } }), + .immediate, + .load_frame, + => { + assert(cond_ty.abiSize(pt.zcu) <= 8); + const tmp_reg = try cg.allocReg(cond_ty, null); + try cg.genCopyToReg(.fromByteSize(cond_ty.abiSize(zcu)), tmp_reg, cond_mcv, .{}); + break :gen_br try cg.asmBr(null, .{ .eq = .{ tmp_reg, .zero } }); + }, + else => unreachable, + }; + + for (liveness_cond_br.then_deaths) |death| try cg.resolveInst(death).die(cg, death); + try cg.genBodyBlock(then_body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + cg.performReloc(reloc); + for (liveness_cond_br.else_deaths) |death| try cg.resolveInst(death).die(cg, death); + try cg.genBodyBlock(else_body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); +} + +fn airBitCast(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const dst_ty = ty_op.ty.toType(); + const src_ty = cg.typeOf(ty_op.operand); + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ty_op.operand})); + + // case 1: no operation needed + // src and dst must have the same ABI size + if (try sel.match(.{ + .requirement = src_ty.abiSize(zcu) == dst_ty.abiSize(zcu), + .patterns = &.{.{ .srcs = &.{.any} }}, + })) { + const src = sel.ops[0]; + if (cg.liveness.operandDies(inst, 0)) { + cg.reused_operands.set(0); + try src.finish(inst, &.{src}, cg); + } else { + const dst = try cg.tempAlloc(dst_ty, .{}); + try src.copy(dst, cg, dst_ty); + try dst.finish(inst, &.{src}, cg); + } + } else return sel.fail(); +} + +fn airIntCast(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const dst_ty = ty_op.ty.toType(); + const src_ty = cg.typeOf(ty_op.operand); + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ty_op.operand})); + + assert(src_ty.isAbiInt(zcu) and dst_ty.isAbiInt(zcu)); + const src_ty_info = src_ty.intInfo(zcu); + const dst_ty_info = dst_ty.intInfo(zcu); + const src_reg_size = (src_ty_info.bits + 63) / 64; + const dst_reg_size = (dst_ty_info.bits + 63) / 64; + const src_tail_bits: u6 = @intCast(src_ty_info.bits % 64); + const dst_tail_bits: u6 = @intCast(dst_ty_info.bits % 64); + + // case 1: no operation needed + // 1. types with same bit size + // 2. sign-extending modes are the same + // 3. i32 to i32...64 promotion + if (try sel.match(.{ + .requirement = src_reg_size == dst_reg_size and + (src_ty_info.bits == dst_ty_info.bits or + (src_tail_bits <= 32 and dst_tail_bits <= 32 and dst_tail_bits != 0) or + (src_tail_bits > 32 and dst_tail_bits > 32) or + (src_tail_bits == 32 and (dst_tail_bits >= 32 or dst_tail_bits == 0))), + .patterns = &.{.{ .srcs = &.{.any} }}, + })) { + const src = sel.ops[0]; + if (cg.liveness.operandDies(inst, 0)) { + cg.reused_operands.set(0); + try src.finish(inst, &.{src}, cg); + } else { + try (try cg.tempInit(dst_ty, .{ .air_ref = ty_op.operand })).finish(inst, &.{src}, cg); + } + } else + // case 2: zero-extend the highest limb + // 1. u32 to u/i33...64 promotion + if (try sel.match(.{ + .required_features = .init(.{ .la64 = true }), + .requirement = src_reg_size == dst_reg_size and + src_ty_info.bits == 32 and + src_ty_info.signedness == .unsigned and + (dst_tail_bits > 32 or dst_tail_bits == 0), + .patterns = &.{.{ .srcs = &.{.any} }}, + })) { + const src = sel.ops[0]; + const dst, const dst_reused = try cg.tempReuseOrAlloc(inst, src, 0, dst_ty, .{}); + if (!dst_reused) + try src.copy(dst, cg, src_ty); + const src_limb, const src_alloc = try src.toLimbValue(src.getLimbCount(cg) - 1, cg).ensureReg(cg, inst, .usize); + const dst_limb, const dst_alloc = try dst.toLimbValue(dst.getLimbCount(cg) - 1, cg).ensureReg(cg, inst, .usize); + + try cg.asmInst(.bstrpick_d(dst_limb, src_limb, 0, src_tail_bits)); + + if (src_alloc) try cg.freeReg(src_limb); + if (dst_alloc) try cg.freeReg(dst_limb); + try dst.finish(inst, &.{src}, cg); + } else return sel.fail(); +} + +/// Checks if a error union value is error. Returns only register values. +fn genIsErr(cg: *CodeGen, eu: Temp, reuse: bool, inverted: bool) !Temp { + var sel = Select.init(cg, null, &.{eu}); + + // case 1: error is in register + if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{.regs} }, + .{ .srcs = &.{.reg_frame} }, + }, + })) { + const src = sel.ops[0]; + const src_mcv = src.tracking(cg); + const err_reg = src_mcv.getRegs()[0]; + const dst = dst: { + if (reuse) { + try src.die(cg); + break :dst try cg.tempInit(.bool, .{ .register = err_reg }); + } else break :dst try cg.tempAlloc(.bool, .{ .use_frame = false }); + }; + const dst_reg = dst.getReg(cg); + + try cg.asmInst(.sltui(dst_reg, err_reg, 1)); + if (!inverted) + try cg.asmInst(.xori(dst_reg, dst_reg, 1)); + + try sel.finish(dst); + return dst; + } + // case 2: error is in memory + else if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{.to_mem} }, + }, + })) { + const src = sel.ops[0]; + var limb = try src.getLimb(.bool, 0, cg, reuse); + while (try limb.moveToRegister(cg, .int, true)) {} + limb.toType(cg, .bool); + const limb_reg = limb.getReg(cg); + + try cg.asmInst(.sltui(limb_reg, limb_reg, 1)); + if (!inverted) + try cg.asmInst(.xori(limb_reg, limb_reg, 1)); + + try sel.finish(limb); + return limb; + } else return sel.fail(); +} + +fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, inverted: bool) !void { + const un_op = cg.getAirData(inst).un_op; + const ops = try cg.tempsFromOperands(inst, .{un_op}); + const reuse = !cg.liveness.operandDies(inst, 0); + const dst = try cg.genIsErr(ops[0], reuse, inverted); + try dst.finish(inst, &ops, cg); +} + +fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = cg.getAirData(inst).ty_op; + const op = (try cg.tempsFromOperands(inst, .{ty_op.operand}))[0]; + const ty = ty_op.ty.toType(); + if (ty.isSlice(cg.pt.zcu)) { + const ptr_ty = ty_op.ty.toType().slicePtrFieldType(cg.pt.zcu); + const dst = try op.getLimb(ptr_ty, 0, cg, cg.liveness.operandDies(inst, 0)); + try dst.finish(inst, &.{op}, cg); + } else { + try op.finish(inst, &.{op}, cg); + } +} + +fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = cg.getAirData(inst).ty_op; + const op = (try cg.tempsFromOperands(inst, .{ty_op.operand}))[0]; + const dst = try op.getLimb(.usize, 1, cg, cg.liveness.operandDies(inst, 0)); + try dst.finish(inst, &.{op}, cg); +} + +fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_pl = cg.getAirData(inst).ty_pl; + const bin = cg.air.extraData(Air.Bin, ty_pl.payload); + const elem_ty = ty_pl.ty.toType().elemType2(zcu); + var sel = Select.init(cg, inst, &try cg.tempsFromOperands(inst, .{ bin.data.lhs, bin.data.rhs })); + const elem_off = elem_ty.abiAlignment(zcu).forward(elem_ty.abiSize(zcu)); + + // case 1: constant index + if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .imm } }, + }, + })) { + const ptr_temp, const index_temp = sel.ops[0..2].*; + ptr_temp.toOffset(cg, @intCast(index_temp.getUnsignedImm(cg) * elem_off)); + try sel.finish(ptr_temp); + } + // case 2: non-constant index + else if (try sel.match(.{ + .required_features = .init(.{ .la64 = true }), + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .to_int_reg } }, + }, + .temps = &.{ + .any_usize_reg, + }, + })) { + const ptr_temp, const index_temp = sel.ops[0..2].*; + const dst = sel.temps[0]; + const dst_reg = dst.getReg(cg); + + try cg.asmInst(.ori(dst_reg, .zero, @intCast(elem_off))); + try cg.asmInst(.mul_d(dst_reg, dst_reg, index_temp.getReg(cg))); + try cg.asmInst(.add_d(dst_reg, dst_reg, ptr_temp.getReg(cg))); + + try sel.finish(dst); + } else return sel.fail(); +} + +fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const bin_op = cg.getAirData(inst).bin_op; + const lhs_ty = cg.typeOf(bin_op.lhs); + const elem_ty = lhs_ty.childType(zcu); + const ptr_ty = if (lhs_ty.isSliceAtRuntime(zcu)) lhs_ty.slicePtrFieldType(zcu) else lhs_ty; + const elem_off = elem_ty.abiAlignment(zcu).forward(elem_ty.abiSize(zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + ops[0] = try ops[0].getLimb(ptr_ty, 0, cg, cg.liveness.operandDies(inst, 0)); + var sel = Select.init(cg, inst, &ops); + + // case 1: constant index + if (try sel.match(.{ + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .imm } }, + }, + .temps = &.{ + .{ .type = elem_ty, .kind = .any }, + }, + })) { + var ptr_temp, const index_temp = sel.ops[0..2].*; + const dst = sel.temps[0]; + ptr_temp.toOffset(cg, @intCast(index_temp.getUnsignedImm(cg) * elem_off)); + try ptr_temp.loadTo(dst, cg, elem_ty, .{}); + try sel.finish(dst); + } + // case 2: non-constant index + else if (try sel.match(.{ + .required_features = .init(.{ .la64 = true }), + .patterns = &.{ + .{ .srcs = &.{ .to_int_reg, .to_int_reg } }, + }, + .temps = &.{ + .any_usize_reg, + .{ .type = elem_ty, .kind = .any }, + }, + })) { + const ptr_temp, const index_temp = sel.ops[0..2].*; + var off, const dst = sel.temps[0..2].*; + const off_reg = off.getReg(cg); + + try cg.asmInst(.ori(off_reg, .zero, @intCast(elem_off))); + try cg.asmInst(.mul_d(off_reg, off_reg, index_temp.getReg(cg))); + try cg.asmInst(.add_d(off_reg, off_reg, ptr_temp.getReg(cg))); + + try off.loadTo(dst, cg, elem_ty, .{}); + + try sel.finish(dst); + } else return sel.fail(); +} + +fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; + const ops = try cg.tempsFromOperands(inst, .{struct_field.struct_operand}); + const off = cg.fieldOffset( + cg.typeOf(struct_field.struct_operand), + ty_pl.ty.toType(), + struct_field.field_index, + ); + const dst, _ = try ops[0].ensureOffsetable(cg); + dst.toOffset(cg, off); + try dst.finish(inst, &ops, cg); +} + +fn airStructFieldPtrConst(cg: *CodeGen, inst: Air.Inst.Index, index: u32) !void { + const ty_op = cg.getAirData(inst).ty_op; + const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const off = cg.fieldOffset(cg.typeOf(ty_op.operand), ty_op.ty.toType(), index); + const dst, _ = try ops[0].ensureOffsetable(cg); + dst.toOffset(cg, off); + try dst.finish(inst, &ops, cg); +} + +fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const payload_ty = ty_op.ty.toType(); + const field_off: i32 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const dst = if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) + try ops[0].read(cg, payload_ty, .{ .off = field_off }) + else + try cg.tempInit(payload_ty, .none); + try dst.finish(inst, &ops, cg); +} + +fn airUnwrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const eu_ty = cg.typeOf(ty_op.operand); + const eu_err_ty = ty_op.ty.toType(); + const payload_ty = eu_ty.errorUnionPayload(zcu); + const field_off: i32 = @intCast(codegen.errUnionErrorOffset(payload_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const dst = try ops[0].read(cg, eu_err_ty, .{ .off = field_off }); + try dst.finish(inst, &ops, cg); +} + +fn airUnwrapErrUnionPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const dst, _ = try ops[0].ensureOffsetable(cg); + dst.toOffset(cg, eu_pl_off); + try dst.finish(inst, &ops, cg); +} + +fn airUnwrapErrUnionErrPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); + const eu_err_ty = ty_op.ty.toType(); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const dst = try ops[0].load(cg, eu_err_ty, .{ .off = eu_err_off }); + try dst.finish(inst, &ops, cg); +} + +fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const eu_ty = ty_op.ty.toType(); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = cg.typeOf(ty_op.operand); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var eu = try cg.tempAlloc(eu_ty, .{}); + try eu.write(ops[0], cg, .{ .off = eu_pl_off }); + + var err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 }); + try eu.write(err, cg, .{ .off = eu_err_off }); + try err.die(cg); + + try eu.finish(inst, &ops, cg); +} + +fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = cg.pt.zcu; + const ty_op = cg.getAirData(inst).ty_op; + const eu_ty = ty_op.ty.toType(); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + + var eu = try cg.tempAlloc(eu_ty, .{}); + try eu.write(ops[0], cg, .{ .off = eu_err_off }); + try eu.finish(inst, &ops, cg); +} + +fn airTry(cg: *CodeGen, inst: Air.Inst.Index) !void { + const pl_op = cg.getAirData(inst).pl_op; + const extra = cg.air.extraData(Air.Try, pl_op.payload); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); + + const operand_ty = cg.typeOf(pl_op.operand); + const result = try cg.genTry(inst, pl_op.operand, operand_ty, false, body); + try result.finish(inst, &.{}, cg); +} + +fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = cg.getAirData(inst).ty_pl; + const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); + + const operand_ty = cg.typeOf(extra.data.ptr); + const result = try cg.genTry(inst, extra.data.ptr, operand_ty, true, body); + try result.finish(inst, &.{}, cg); +} + +fn genTry( + cg: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + operand_ty: Type, + operand_is_ptr: bool, + body: []const Air.Inst.Index, +) !Temp { + const zcu = cg.pt.zcu; + const liveness_cond_br = cg.liveness.getCondBr(inst); + const ops = try cg.tempsFromOperands(inst, .{operand}); + const reuse_op = !cg.liveness.operandDies(inst, 0); + + const is_err_temp = if (operand_is_ptr) + unreachable // TODO + else + try cg.genIsErr(ops[0], reuse_op, true); + const is_err_reg = is_err_temp.getReg(cg); + + if (!reuse_op) + try ops[0].die(cg); + try is_err_temp.die(cg); + try cg.resetTemps(@enumFromInt(0)); + + const reloc = try cg.asmBr(null, .{ .ne = .{ is_err_reg, .zero } }); + const state = try cg.saveState(); + for (liveness_cond_br.else_deaths) |death| try cg.resolveInst(death).die(cg, death); + try cg.genBodyBlock(body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + cg.performReloc(reloc); + + for (liveness_cond_br.then_deaths) |death| try cg.resolveInst(death).die(cg, death); + + const payload_ty = operand_ty.errorUnionPayload(zcu); + const field_off: i32 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu)); + const result = + if (cg.liveness.isUnused(inst)) + try cg.tempInit(payload_ty, .unreach) + else if (operand_is_ptr) + unreachable // TODO + else if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) + try ops[0].read(cg, payload_ty, .{ .off = field_off }) + else + try cg.tempInit(payload_ty, .none); + + return result; +} diff --git a/src/arch/loongarch/Emit.zig b/src/arch/loongarch/Emit.zig new file mode 100644 index 000000000000..5b84d66d0fa7 --- /dev/null +++ b/src/arch/loongarch/Emit.zig @@ -0,0 +1,233 @@ +//! This file contains the functionality for emitting LoongArch MIR as machine code + +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.emit); +const R_LARCH = std.elf.R_LARCH; + +const link = @import("../../link.zig"); +const codegen = @import("../../codegen.zig"); +const Zcu = @import("../../Zcu.zig"); +const Type = @import("../../Type.zig"); +const InternPool = @import("../../InternPool.zig"); + +const Lower = @import("Lower.zig"); +const Lir = @import("Lir.zig"); +const Mir = @import("Mir.zig"); +const bits = @import("bits.zig"); + +const Emit = @This(); + +lower: Lower, +pt: Zcu.PerThread, +bin_file: *link.File, +owner_nav: InternPool.Nav.Index, +atom_index: u32, +debug_output: link.File.DebugInfoOutput, +code: *std.ArrayListUnmanaged(u8), + +prev_di_loc: Loc, +/// Relative to the beginning of `code`. +prev_di_pc: usize, + +pub const Error = Lower.Error || error{EmitFail} || link.File.UpdateDebugInfoError; + +pub fn emitMir(emit: *Emit) Error!void { + const ip = &emit.pt.zcu.intern_pool; + log.debug("Begin Emit: {}", .{ip.getNav(emit.owner_nav).fqn.fmt(ip)}); + const gpa = emit.lower.link_file.comp.gpa; + + const code_offset_mapping = try emit.lower.allocator.alloc(u32, emit.lower.mir.instructions.len); + defer emit.lower.allocator.free(code_offset_mapping); + var relocs: std.ArrayListUnmanaged(Reloc) = .empty; + defer relocs.deinit(emit.lower.allocator); + + for (0..emit.lower.mir.instructions.len) |mir_i| { + const mir_index: Mir.Inst.Index = @intCast(mir_i); + const mir_inst = emit.lower.mir.instructions.get(mir_index); + code_offset_mapping[mir_index] = @intCast(emit.code.items.len); + + const lowered = try emit.lower.lowerMir(mir_index); + var lir_relocs = lowered.relocs; + + if (mir_inst.tag == Mir.Inst.Tag.fromPseudo(.func_epilogue)) { + switch (emit.debug_output) { + .dwarf => |dwarf| { + try dwarf.setEpilogueBegin(); + try emit.dbgAdvancePCAndLine(emit.prev_di_loc); + }, + .plan9, .none => {}, + } + } + + for (lowered.insts, 0..) |lir_inst, lir_index| { + const start_offset: u32 = @intCast(emit.code.items.len); + try emit.code.writer(gpa).writeInt(u32, lir_inst.encode(), .little); + + while (lir_relocs.len > 0 and + lir_relocs[0].lir_index == lir_index) : ({ + lir_relocs = lir_relocs[1..]; + }) switch (lir_relocs[0].target) { + .inst => |reloc| { + try relocs.append(emit.lower.allocator, .{ + .source = start_offset, + .loc = reloc.loc, + .target = reloc.inst, + .off = lir_relocs[0].off, + }); + }, + .elf_nav => |sym| { + const sym_index = switch (try codegen.genNavRef( + emit.bin_file, + emit.pt, + emit.lower.src_loc, + sym.symbol, + emit.lower.target, + )) { + .sym_index => |index| index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + }; + const elf_file = emit.lower.link_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + const atom_ptr = zo.symbol(emit.atom_index).atom(elf_file).?; + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset, + .r_info = (@as(u64, sym_index) << 32) | @intFromEnum(sym.ty), + .r_addend = lir_relocs[0].off, + }, zo); + }, + .elf_uav => |sym| { + const sym_index = switch (try emit.bin_file.lowerUav( + emit.pt, + sym.symbol.val, + Type.fromInterned(sym.symbol.orig_ty).ptrAlignment(emit.pt.zcu), + emit.lower.src_loc, + )) { + .sym_index => |index| index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + }; + const elf_file = emit.lower.link_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + const atom_ptr = zo.symbol(emit.atom_index).atom(elf_file).?; + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset, + .r_info = (@as(u64, sym_index) << 32) | @intFromEnum(sym.ty), + .r_addend = lir_relocs[0].off, + }, zo); + }, + }; + } + std.debug.assert(lir_relocs.len == 0); + + switch (mir_inst.tag.unwrap()) { + else => {}, + .pseudo => |tag| switch (tag) { + .func_prologue => { + switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.setPrologueEnd(), + .plan9, .none => {}, + } + }, + .dbg_line_stmt_line_column => try emit.dbgAdvancePCAndLine(.{ + .line = mir_inst.data.line_column.line, + .column = mir_inst.data.line_column.column, + .is_stmt = true, + }), + .dbg_line_line_column => try emit.dbgAdvancePCAndLine(.{ + .line = mir_inst.data.line_column.line, + .column = mir_inst.data.line_column.column, + .is_stmt = false, + }), + .dbg_enter_block => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.enterBlock(emit.code.items.len), + .plan9, .none => {}, + }, + .dbg_exit_block => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.leaveBlock(emit.code.items.len), + .plan9, .none => {}, + }, + .dbg_enter_inline_func => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.enterInlineFunc( + mir_inst.data.func, + emit.code.items.len, + emit.prev_di_loc.line, + emit.prev_di_loc.column, + ), + .plan9, .none => {}, + }, + .dbg_exit_inline_func => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.leaveInlineFunc( + mir_inst.data.func, + emit.code.items.len, + ), + .plan9, .none => {}, + }, + else => {}, + }, + } + } + + for (relocs.items) |reloc| { + const target = @as(i32, @intCast(code_offset_mapping[reloc.target])) + reloc.off; + const offset = target - @as(i32, @intCast(reloc.source)); + const offset_u32 = @as(u32, @bitCast(offset)); + + const inst_bytes = emit.code.items[reloc.source..][0..4]; + var inst_u32 = std.mem.readInt(u32, inst_bytes, .little); + + switch (reloc.loc) { + .b26 => inst_u32 = inst_u32 | + (@as(u32, @as(u16, @truncate((offset_u32 >> 2) & 0xffff))) << 10) | + @as(u32, @as(u10, @truncate((offset_u32 >> 2) >> 16))), + .k16 => inst_u32 = inst_u32 | + (@as(u32, @as(u16, @truncate((offset_u32 >> 2) & 0xffff))) << 10), + } + + std.mem.writeInt(u32, inst_bytes, inst_u32, .little); + } + log.debug("End Emit: {}", .{ip.getNav(emit.owner_nav).fqn.fmt(ip)}); +} + +fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { + @branchHint(.cold); + assert(emit.lower.err_msg == null); + emit.lower.err_msg = try Zcu.ErrorMsg.create(emit.lower.allocator, emit.lower.src_loc, format, args); + return error.LowerFail; +} + +const Reloc = struct { + source: usize, + loc: Lower.Reloc.Type, + target: Mir.Inst.Index, + off: i32, +}; + +const Loc = struct { + line: u32, + column: u32, + is_stmt: bool, +}; + +fn dbgAdvancePCAndLine(emit: *Emit, loc: Loc) Error!void { + const delta_line = @as(i33, loc.line) - @as(i33, emit.prev_di_loc.line); + const delta_pc: usize = emit.code.items.len - emit.prev_di_pc; + log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line }); + switch (emit.debug_output) { + .dwarf => |dwarf| { + if (loc.is_stmt != emit.prev_di_loc.is_stmt) try dwarf.negateStmt(); + if (loc.column != emit.prev_di_loc.column) try dwarf.setColumn(loc.column); + try dwarf.advancePCAndLine(delta_line, delta_pc); + emit.prev_di_loc = loc; + emit.prev_di_pc = emit.code.items.len; + }, + .plan9, .none => {}, + } +} diff --git a/src/arch/loongarch/Lir.zig b/src/arch/loongarch/Lir.zig new file mode 100644 index 000000000000..3c5e0bdcd34e --- /dev/null +++ b/src/arch/loongarch/Lir.zig @@ -0,0 +1,162 @@ +//! Lower Intermediate Representation. +//! This IR have 1:1 correspondence with machine instructions, +//! while keeping instruction encoding unknown. +//! The purpose is to keep the lowering process unaware of instruction format. + +const std = @import("std"); +const expectEqual = std.testing.expectEqual; +const expectEqualStrings = std.testing.expectEqualStrings; + +const encoding = @import("encoding.zig"); +const bits = @import("bits.zig"); +const Register = bits.Register; + +pub const Inst = struct { + opcode: encoding.OpCode, + data: encoding.Data, + + pub fn encode(inst: Inst) u32 { + const opcode = inst.opcode.enc(); + return opcode | inst.data.enc(); + } + + pub fn format( + inst: Inst, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + try writer.print("{s}", .{@tagName(inst.opcode)}); + inline for (@typeInfo(encoding.Format).@"enum".fields) |enum_field| { + if (enum_field.value == @intFromEnum(std.meta.activeTag(inst.data))) { + const data_ty = @FieldType(encoding.Data, enum_field.name); + const data_ty_info = @typeInfo(data_ty); + switch (data_ty_info) { + .void => {}, + .@"struct" => |struct_info| { + inline for (struct_info.fields) |field| { + const data_val = @field(@field(inst.data, enum_field.name), field.name); + if (field.type == Register) { + try writer.print(" {s}", .{@tagName(data_val)}); + } else { + switch (@typeInfo(field.type)) { + .int => try writer.print(" {}", .{data_val}), + else => unreachable, + } + } + } + }, + else => unreachable, + } + } + } + } + + pub fn fromInst(inst: encoding.Inst) Inst { + return .{ .opcode = inst.opcode, .data = inst.data }; + } +}; + +test "instruction encoding" { + try expectEqual(0x02c02808, (Inst{ .opcode = .addi_d, .data = .{ + .DJSk12 = .{ .r8, .r0, 10 }, + } }).encode()); + try expectEqual(0x01140841, (Inst{ .opcode = .fabs_d, .data = .{ + .DJ = .{ .f1, .f2 }, + } }).encode()); + try expectEqual(0x002a0000, (Inst{ .opcode = .@"break", .data = .{ + .Ud15 = .{0}, + } }).encode()); + try expectEqual(0x00160c41, (Inst{ .opcode = .orn, .data = .{ + .DJK = .{ .r1, .r2, .r3 }, + } }).encode()); + try expectEqual(0x0c100820, (Inst{ .opcode = .fcmp_caf_s, .data = .{ + .DJK = .{ .fcc0, .f1, .f2 }, + } }).encode()); + try expectEqual(0x0c100820, Inst.fromInst(.fcmp_caf_s(.fcc0, .f1, .f2)).encode()); + try expectEqual(0x06483800, Inst.fromInst(.eret()).encode()); +} + +test "instruction formatting" { + try expectEqualStrings("addi_d r8 r0 10", std.fmt.comptimePrint("{}", .{Inst{ .opcode = .addi_d, .data = .{ + .DJSk12 = .{ .r8, .r0, 10 }, + } }})); + try expectEqualStrings("fabs_d f1 f2", std.fmt.comptimePrint("{}", .{Inst{ .opcode = .fabs_d, .data = .{ + .DJ = .{ .f1, .f2 }, + } }})); + try expectEqualStrings("break 0", std.fmt.comptimePrint("{}", .{Inst{ .opcode = .@"break", .data = .{ + .Ud15 = .{0}, + } }})); + try expectEqualStrings("fcmp_caf_s fcc0 f1 f2", std.fmt.comptimePrint("{}", .{Inst{ .opcode = .fcmp_caf_s, .data = .{ + .DJK = .{ .fcc0, .f1, .f2 }, + } }})); +} + +pub const MemOp = enum { store, load }; + +pub const SizedMemOp = struct { + op: MemOp, + size: bits.Memory.Size, + signedness: std.builtin.Signedness, + + pub fn toOpCodeRI(op: SizedMemOp) encoding.OpCode { + return switch (op.op) { + .store => switch (op.size) { + .byte => .st_b, + .hword => .st_h, + .word => .st_w, + .dword => .st_d, + }, + .load => switch (op.size) { + .byte => switch (op.signedness) { + .signed => .ld_b, + .unsigned => .ld_bu, + }, + .hword => switch (op.signedness) { + .signed => .ld_h, + .unsigned => .ld_hu, + }, + .word => switch (op.signedness) { + .signed => .ld_w, + .unsigned => .ld_wu, + }, + .dword => .ld_d, + }, + }; + } + + pub fn toOpCodeRR(op: SizedMemOp) encoding.OpCode { + return switch (op.op) { + .store => switch (op.size) { + .byte => .stx_b, + .hword => .stx_h, + .word => .stx_w, + .dword => .stx_d, + }, + .load => switch (op.size) { + .byte => switch (op.signedness) { + .signed => .ldx_b, + .unsigned => .ldx_bu, + }, + .hword => switch (op.signedness) { + .signed => .ldx_h, + .unsigned => .ldx_hu, + }, + .word => switch (op.signedness) { + .signed => .ldx_w, + .unsigned => .ldx_wu, + }, + .dword => .ldx_d, + }, + }; + } + + pub fn format( + op: SizedMemOp, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + try writer.print("{s}", .{@tagName(op.toOpCodeRI())}); + } +}; diff --git a/src/arch/loongarch/Lower.zig b/src/arch/loongarch/Lower.zig new file mode 100644 index 000000000000..783a21435625 --- /dev/null +++ b/src/arch/loongarch/Lower.zig @@ -0,0 +1,442 @@ +//! This file contains the functionality for lowering LoongArch MIR to Instructions + +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.lower); + +const Allocator = std.mem.Allocator; +const cast = std.math.cast; +const R_LARCH = std.elf.R_LARCH; + +const link = @import("../../link.zig"); +const Air = @import("../../Air.zig"); +const Zcu = @import("../../Zcu.zig"); +const InternPool = @import("../../InternPool.zig"); +const codegen = @import("../../codegen.zig"); +const ErrorMsg = Zcu.ErrorMsg; + +const Mir = @import("Mir.zig"); +const abi = @import("abi.zig"); +const bits = @import("bits.zig"); +const encoding = @import("encoding.zig"); +const Lir = @import("Lir.zig"); +const utils = @import("./utils.zig"); +const Register = bits.Register; + +const Lower = @This(); + +pt: Zcu.PerThread, +link_file: *link.File, +target: *const std.Target, +output_mode: std.builtin.OutputMode, +link_mode: std.builtin.LinkMode, +pic: bool, +allocator: std.mem.Allocator, +mir: Mir, +cc: std.builtin.CallingConvention, +err_msg: ?*Zcu.ErrorMsg = null, +src_loc: Zcu.LazySrcLoc, +result_insts_len: ResultInstIndex = undefined, +result_insts: [max_result_insts]Lir.Inst = undefined, +result_relocs_len: ResultRelocIndex = undefined, +result_relocs: [max_result_relocs]Reloc = undefined, + +pub const Error = error{ + OutOfMemory, + LowerFail, + CodegenFail, +} || codegen.GenerateSymbolError; + +/// Lowered relocation. +/// The fields in instructions to be relocated must be lowered to zero. +pub const Reloc = struct { + lir_index: u8, + target: Target, + off: i32, + + pub const Target = union(enum) { + inst: struct { + loc: Type, + inst: Mir.Inst.Index, + }, + elf_nav: struct { + ty: R_LARCH, + symbol: InternPool.Nav.Index, + }, + elf_uav: struct { + ty: R_LARCH, + symbol: InternPool.Key.Ptr.BaseAddr.Uav, + }, + + pub fn format( + target: Target, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (target) { + .inst => |pl| try writer.print("MIR: {} ({s})", .{ pl.inst, @tagName(pl.loc) }), + .elf_nav => |pl| try writer.print("NAV: R_LARCH_{s} => {}", .{ @tagName(pl.ty), pl.symbol }), + .elf_uav => |pl| try writer.print("UAV: R_LARCH_{s} => {}", .{ @tagName(pl.ty), pl.symbol }), + } + } + }; + + pub const Type = enum { + /// Immediate slot of Sd10k16ps2, right shift 2, relative to PC + b26, + /// Immediate slot of JDSk16ps2, right shift 2, relative to PC + k16, + }; +}; + +/// The returned slice is overwritten by the next call to lowerMir. +pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { + insts: []const Lir.Inst, + relocs: []const Reloc, +} { + lower.result_insts = undefined; + lower.result_relocs = undefined; + errdefer lower.result_insts = undefined; + errdefer lower.result_relocs = undefined; + lower.result_insts_len = 0; + lower.result_relocs_len = 0; + defer lower.result_insts_len = undefined; + defer lower.result_relocs_len = undefined; + + const inst = lower.mir.instructions.get(index); + log.debug(" {}: {}", .{ index, inst }); + + lower_inst: switch (inst.tag.unwrap()) { + .inst => |opcode| lower.emitLir(.{ .opcode = opcode, .data = inst.data.op }), + .pseudo => |tag| { + switch (tag) { + .none => {}, + // TODO: impl func prolugue + .func_prologue => { + if (lower.mir.frame_size != 0) { + const off = -@as(i64, @intCast(lower.mir.frame_size)); + try lower.emitRegBiasToReg(.sp, .sp, off); + } + if (lower.mir.spill_ra) + try lower.emitRegFrameOp(.ra, .{ .index = .ret_addr_frame }, .t0, .st_d, .stx_d); + }, + .func_epilogue => { + if (lower.mir.spill_ra) + try lower.emitRegFrameOp(.ra, .{ .index = .ret_addr_frame }, .t0, .ld_d, .ldx_d); + if (lower.mir.frame_size != 0) + try lower.emitRegBiasToReg(.sp, .sp, @intCast(lower.mir.frame_size)); + lower.emit(.jirl(.ra, .ra, 0)); + }, + .jump_to_epilogue => { + if (index + 1 == lower.mir.epilogue_begin) { + log.debug("omitted jump_to_epilogue", .{}); + } else { + lower.emit(.b(0, 0)); + lower.relocInst(.b26, lower.mir.epilogue_begin, 0); + } + }, + .dbg_line_line_column, + .dbg_line_stmt_line_column, + .dbg_enter_block, + .dbg_exit_block, + .dbg_enter_inline_func, + .dbg_exit_inline_func, + => {}, + .branch => { + const target_inst = inst.data.br.inst; + if (target_inst + 1 == index) break :lower_inst; + switch (inst.data.br.cond) { + .none => { + lower.emit(.b(0, 0)); + lower.relocInst(.b26, target_inst, 0); + }, + .eq => |regs| { + lower.emit(.beq(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + .ne => |regs| { + lower.emit(.bne(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + .le => |regs| { + lower.emit(.ble(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + .gt => |regs| { + lower.emit(.bgt(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + .leu => |regs| { + lower.emit(.bleu(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + .gtu => |regs| { + lower.emit(.bgtu(regs[0], regs[1], 0)); + lower.relocInst(.k16, target_inst, 0); + }, + } + }, + .imm_to_reg => try lower.emitImmToReg(inst.data.imm_reg.imm, inst.data.imm_reg.reg), + .frame_addr_to_reg => { + const frame_addr = inst.data.frame_reg.frame; + const frame_loc = lower.resolveFrame(frame_addr.index); + const reg = inst.data.frame_reg.reg; + try lower.emitRegBiasToReg(reg, frame_loc.base, @as(i64, frame_loc.offset + frame_addr.off)); + }, + .frame_addr_reg_mem => { + const data = inst.data.memop_frame_reg; + try lower.emitRegFrameOp(data.reg, data.frame, data.tmp_reg, data.op.toOpCodeRI(), data.op.toOpCodeRR()); + }, + .nav_memop => { + const data = inst.data.memop_nav_reg; + lower.emit(.pcalau12i(data.tmp_reg, 0)); + lower.relocElfNav(.PCALA_HI20, data.nav); + lower.emit(.{ .opcode = data.op.toOpCodeRI(), .data = .{ .DJSk12 = .{ data.reg, data.tmp_reg, 0 } } }); + lower.relocElfNav(.PCALA_LO12, data.nav); + }, + .uav_memop => { + const data = inst.data.memop_uav_reg; + lower.emit(.pcalau12i(data.tmp_reg, 0)); + lower.relocElfUav(.PCALA_HI20, data.uav); + lower.emit(.{ .opcode = data.op.toOpCodeRI(), .data = .{ .DJSk12 = .{ data.reg, data.tmp_reg, 0 } } }); + lower.relocElfUav(.PCALA_LO12, data.uav); + }, + .nav_addr_to_reg => { + const data = inst.data.nav_reg; + lower.emit(.pcalau12i(data.reg, 0)); + lower.relocElfNav(.PCALA_HI20, data.nav); + lower.emit(.addi_d(data.reg, data.reg, 0)); + lower.relocElfNav(.PCALA_LO12, data.nav); + }, + .uav_addr_to_reg => { + const data = inst.data.uav_reg; + lower.emit(.pcalau12i(data.reg, 0)); + lower.relocElfUav(.PCALA_HI20, data.uav); + lower.emit(.addi_d(data.reg, data.reg, 0)); + lower.relocElfUav(.PCALA_LO12, data.uav); + }, + .call => { + if (!lower.hasFeature(.@"64bit")) return lower.fail("TODO function call in LA32", .{}); + const nav = inst.data.nav; + lower.emit(.pcaddu18i(.ra, 0)); + lower.relocElfNav(.CALL36, nav); + lower.emit(.jirl(.ra, .ra, 0)); + }, + .load_ra => if (lower.mir.spill_ra) + try lower.emitRegFrameOp(.ra, .{ .index = .ret_addr_frame }, .ra, .ld_d, .ldx_d), + .spill_int_regs => lower.emitRegSpill(inst.data.reg_list, .{ + .frame = .spill_int_frame, + .reg_class = .int, + .inst = .st_d, + }), + .restore_int_regs => lower.emitRegSpill(inst.data.reg_list, .{ + .frame = .spill_int_frame, + .reg_class = .int, + .inst = .ld_d, + }), + .spill_float_regs => lower.emitRegSpill(inst.data.reg_list, .{ + .frame = .spill_float_frame, + .reg_class = .float, + .inst = .fst_d, + }), + .restore_float_regs => lower.emitRegSpill(inst.data.reg_list, .{ + .frame = .spill_float_frame, + .reg_class = .float, + .inst = .fld_d, + }), + } + }, + } + + return .{ + .insts = lower.result_insts[0..lower.result_insts_len], + .relocs = lower.result_relocs[0..lower.result_relocs_len], + }; +} + +inline fn emit(lower: *Lower, inst: encoding.Inst) void { + lower.emitLir(.fromInst(inst)); +} + +fn emitLir(lower: *Lower, inst: Lir.Inst) void { + log.debug(" | {}", .{inst}); + lower.result_insts[lower.result_insts_len] = inst; + lower.result_insts_len += 1; +} + +fn reloc(lower: *Lower, target: Reloc.Target, off: i32) void { + log.debug(" | reloc: {} (+{})", .{ target, off }); + lower.result_relocs[lower.result_relocs_len] = .{ + .lir_index = lower.result_insts_len - 1, + .target = target, + .off = off, + }; + lower.result_relocs_len += 1; +} + +inline fn relocElfNav(lower: *Lower, ty: R_LARCH, sym: bits.NavOffset) void { + lower.reloc(.{ .elf_nav = .{ .ty = ty, .symbol = sym.index } }, sym.off); +} + +inline fn relocElfUav(lower: *Lower, ty: R_LARCH, sym: bits.UavOffset) void { + lower.reloc(.{ .elf_uav = .{ .ty = ty, .symbol = sym.index } }, sym.off); +} + +inline fn relocInst(lower: *Lower, loc: Reloc.Type, inst: Mir.Inst.Index, off: i32) void { + lower.reloc(.{ .inst = .{ .loc = loc, .inst = inst } }, off); +} + +fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { + @branchHint(.cold); + assert(lower.err_msg == null); + lower.err_msg = try ErrorMsg.create(lower.allocator, lower.src_loc, format, args); + return error.LowerFail; +} + +fn hasFeature(lower: *Lower, feature: std.Target.loongarch.Feature) bool { + const target = lower.pt.zcu.getTarget(); + const features = target.cpu.features; + return std.Target.loongarch.featureSetHas(features, feature); +} + +inline fn resolveFrame(lower: *Lower, frame: bits.FrameIndex) Mir.FrameLoc { + return lower.mir.frame_locs.get(@intFromEnum(frame)); +} + +const max_result_insts = @max( + 1, // non-pseudo instructions/branch + abi.zigcc.all_static.len + 1, // push_regs/pop_regs + abi.c_abi.all_static.len + 1, // push_regs/pop_regs + 4, // emitImmToReg/imm_to_reg + 5, // emitRegBiasToReg/frame_addr_to_reg/func_prolugue + 6, // emitRegFrameOp/frame_addr_reg_mem/load_ra + 7, // func_epilogue +); +const max_result_relocs = @max( + 1, // jump_to_epilogue + 2, // call + 0, +); + +const ResultInstIndex = std.math.IntFittingRange(0, max_result_insts); +const ResultRelocIndex = std.math.IntFittingRange(0, max_result_relocs); + +/// Loads an immediate to a reg. +/// Emits up to 5 instructions. +fn emitImmToReg(lower: *Lower, imm: u64, dst: Register) !void { + var use_lu12i = false; + var set_hi = false; + // Loads 31..12 bits as LU12I.W clears 11..00 bits + if (utils.notZero((imm & 0x00000000fffff000) >> 12)) |part| { + lower.emit(.lu12i_w(dst, @truncate(@as(i64, @bitCast(part))))); + use_lu12i = true; + set_hi = (part >> 11) != 0; + } + // Then loads 11..0 bits with ORI first if LU12I.W is not used + // in order to clear 63..12 bits + const lo12: u12 = @truncate(imm & 0x0000000000000fff); + if (!use_lu12i) { + lower.emit(.ori(dst, .zero, lo12)); + set_hi = false; + } + // Loads 51..32 bits + if (utils.notZero((imm & 0x000fffff00000000) >> 32)) |part| { + if (!(part == 0xfffff and set_hi)) { + if (!lower.hasFeature(.@"64bit")) { + return lower.fail("Cannot load immediate with more than 32 bits to integer register in LA32", .{}); + } + lower.emit(.cu32i_d(dst, @truncate(@as(i64, @bitCast(part))))); + set_hi = (part >> 11) != 0; + } + } + // Loads 63..52 bits + if (utils.notZero((imm & 0xfff0000000000000) >> 52)) |part| { + if (!(part == 0xfff and set_hi)) { + if (!lower.hasFeature(.@"64bit")) { + return lower.fail("Cannot load immediate with more than 32 bits to integer register in LA32", .{}); + } + lower.emit(.cu52i_d(dst, dst, @truncate(@as(i64, @bitCast(part))))); + } + } + // Loads 11..0 at the end if LU12I.W is used, to preserve higher bits. + if (use_lu12i and lo12 != 0x000) { + lower.emit(.ori(dst, dst, lo12)); + } +} + +/// Loads an immediate plus a reg to a reg. +/// Emits up to 6 instructions. +/// dst and src cannot be the same reg, or t0 will be clobberred. +fn emitRegBiasToReg(lower: *Lower, dst: Register, src: Register, imm: i64) !void { + if (cast(i12, imm)) |imm12| { + if (lower.hasFeature(.@"64bit")) { + lower.emit(.addi_d(dst, src, imm12)); + } else { + lower.emit(.addi_w(dst, src, imm12)); + } + } else if (dst == src) { + try lower.emitImmToReg(@bitCast(imm), .t0); + if (lower.hasFeature(.@"64bit")) { + lower.emit(.add_d(dst, .t0, src)); + } else { + lower.emit(.add_w(dst, .t0, src)); + } + } else { + try lower.emitImmToReg(@bitCast(imm), dst); + if (lower.hasFeature(.@"64bit")) { + lower.emit(.add_d(dst, dst, src)); + } else { + lower.emit(.add_w(dst, dst, src)); + } + } +} + +/// Emits up to 6 instructions. +/// See Mir.Inst.PseudoTag.frame_addr_reg_mem. +fn emitRegFrameOp( + lower: *Lower, + reg: Register, + frame: bits.FrameAddr, + tmp_reg: Register, + op_ri: encoding.OpCode, + op_rr: encoding.OpCode, +) !void { + const frame_loc = lower.resolveFrame(frame.index); + const offset = @as(i64, frame_loc.offset + frame.off); + if (cast(i12, offset)) |off12| { + lower.emit(.{ + .opcode = op_ri, + .data = .{ .DJSk12 = .{ reg, frame_loc.base, off12 } }, + }); + } else { + try lower.emitImmToReg(@bitCast(@as(i64, frame_loc.offset + frame.off)), tmp_reg); + lower.emit(.{ + .opcode = op_rr, + .data = .{ .DJK = .{ reg, frame_loc.base, tmp_reg } }, + }); + } +} + +const SpillOptions = struct { + frame: bits.FrameIndex, + reg_class: Register.Class, + inst: encoding.OpCode, +}; + +/// Emits register spill/restores with DJSk12/VdJSK12/XdJSK12 instructions. +/// rd/vd/xd is the operand register. rj + si12 is the frame address. +fn emitRegSpill(lower: *Lower, regs: Mir.RegisterList, opts: SpillOptions) void { + const frame = lower.resolveFrame(opts.frame); + var offset = frame.offset; + const reg_size: i32 = @intCast(opts.reg_class.byteSize(lower.target)); + var iter = regs.iterator(.{}); + while (iter.next()) |reg_index| { + const reg = Mir.RegisterList.getRegFromIndex(opts.reg_class, reg_index); + lower.emit(.{ + .opcode = opts.inst, + .data = .{ .DJSk12 = .{ reg, frame.base, @intCast(offset) } }, + }); + offset += reg_size; + } +} diff --git a/src/arch/loongarch/Mir.zig b/src/arch/loongarch/Mir.zig new file mode 100644 index 000000000000..263eed76b7f0 --- /dev/null +++ b/src/arch/loongarch/Mir.zig @@ -0,0 +1,429 @@ +//! Machine Intermediate Representation. +//! This data is produced by CodeGen.zig + +const Mir = @This(); +const std = @import("std"); + +const bits = @import("bits.zig"); +const Register = bits.Register; +const Lir = @import("Lir.zig"); +const encoding = @import("encoding.zig"); +const Emit = @import("Emit.zig"); + +const InternPool = @import("../../InternPool.zig"); +const codegen = @import("../../codegen.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); + +instructions: std.MultiArrayList(Inst).Slice, +frame_locs: std.MultiArrayList(FrameLoc).Slice, +frame_size: usize, +/// The first instruction of epilogue. +epilogue_begin: Inst.Index, +/// Indicates that $ra needs to be spilled. +spill_ra: bool, + +pub const Inst = struct { + tag: Tag, + /// The meaning of this depends on `tag`. + data: Data, + + pub const Index = u32; + + pub const Tag = enum(u16) { + _, + + pub fn fromInst(opcode: encoding.OpCode) Tag { + return @enumFromInt(@intFromEnum(opcode)); + } + + pub fn fromPseudo(tag: PseudoTag) Tag { + return @enumFromInt(@as(u16, @intFromEnum(tag)) | (1 << 15)); + } + + pub fn unwrap(tag: Tag) union(enum) { pseudo: PseudoTag, inst: encoding.OpCode } { + if ((@intFromEnum(tag) & (1 << 15)) != 0) { + return .{ .pseudo = @enumFromInt(@intFromEnum(tag) & ~@as(u16, @intCast((1 << 15)))) }; + } else { + return .{ .inst = @enumFromInt(@intFromEnum(tag)) }; + } + } + }; + + pub const PseudoTag = enum { + /// Placeholder for backpatch or dead. No payload. + none, + + /// Branch instructions, uses `br` payload. + branch, + /// Load an 64-bit immediate to register, uses `imm_reg` payload. + imm_to_reg, + /// Load frame address to register, uses `frame_reg` payload. + frame_addr_to_reg, + /// Frame memory load/store operations, uses `memop_frame_reg` payload. + frame_addr_reg_mem, + /// NAV memory load/store operations, uses `memop_nav_reg` payload. + nav_memop, + /// UAV memory load/store operations, uses `memop_uav_reg` payload. + uav_memop, + /// Load NAV address to register, uses `nav_reg` payload. + nav_addr_to_reg, + /// Load UAV address to register, uses `uav_reg` payload. + uav_addr_to_reg, + /// Function call, uses `nav` payload. + call, + /// Loads spilled $ra back to $ra, no payload. + load_ra, + + /// Prologue of a function, no payload. + func_prologue, + /// Epilogue of a function, no payload. + func_epilogue, + /// Jump to epilogue, no payload. + jump_to_epilogue, + /// Spills general-purpose integer registers, uses `reg_list` payload. + spill_int_regs, + /// Restores general-purpose integer registers, uses `reg_list` payload. + restore_int_regs, + /// Spills general-purpose float registers, uses `reg_list` payload. + spill_float_regs, + /// Restores general-purpose float registers, uses `reg_list` payload. + restore_float_regs, + + /// Update debug line with is_stmt register set, uses `line_column` payload. + dbg_line_stmt_line_column, + /// Update debug line with is_stmt register clear, uses `line_column` payload. + dbg_line_line_column, + /// Start of lexical block, no payload. + dbg_enter_block, + /// End of lexical block, no payload. + dbg_exit_block, + /// Start of inline function block, uses `func` payload. + /// Payload points to the inlined function. + dbg_enter_inline_func, + /// End of inline function block, uses `func` payload. + /// Payload points to the outer function. + dbg_exit_inline_func, + }; + + pub const Data = union { + pub const none = undefined; + + op: encoding.Data, + + /// Register list. + reg_list: RegisterList, + /// Debug line and column position. + line_column: struct { + line: u32, + column: u32, + }, + /// Branches. + br: struct { + inst: Index, + cond: BranchCondition, + }, + /// Immediate and register + imm_reg: struct { + imm: u64, + reg: Register, + }, + /// Frame address and register + frame_reg: struct { + frame: bits.FrameAddr, + reg: Register, + }, + /// Mem op, frame address and register + memop_frame_reg: struct { + op: Lir.SizedMemOp, + frame: bits.FrameAddr, + reg: Register, + tmp_reg: Register, + }, + /// Mem op, NAV offset and register + memop_nav_reg: struct { + op: Lir.SizedMemOp, + nav: bits.NavOffset, + reg: Register, + tmp_reg: Register, + }, + /// Mem op, UAV offset and register + memop_uav_reg: struct { + op: Lir.SizedMemOp, + uav: bits.UavOffset, + reg: Register, + tmp_reg: Register, + }, + /// NAV offset and register + nav_reg: struct { + nav: bits.NavOffset, + reg: Register, + }, + /// UAV offset and register + uav_reg: struct { + uav: bits.UavOffset, + reg: Register, + }, + /// NAV offset + nav: bits.NavOffset, + /// Function index + func: InternPool.Index, + }; + + pub inline fn initInst(inst: encoding.Inst) Inst { + const lir_inst = Lir.Inst.fromInst(inst); + return .{ .tag = .fromInst(lir_inst.opcode), .data = .{ .op = lir_inst.data } }; + } + + pub fn format( + inst: Inst, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (inst.tag.unwrap()) { + .pseudo => |tag| { + switch (tag) { + .dbg_line_stmt_line_column, .dbg_line_line_column => try writer.print(".{s} L{d}:{d}", .{ + @tagName(tag), + inst.data.line_column.line, + inst.data.line_column.column, + }), + .branch => try writer.print(".branch {} => {}", .{ inst.data.br.cond, inst.data.br.inst }), + .imm_to_reg => try writer.print(".imm_to_reg {s} <== {}", .{ + @tagName(inst.data.imm_reg.reg), + inst.data.imm_reg.imm, + }), + .frame_addr_to_reg => try writer.print(".frame_addr_to_reg {s} <== {}", .{ + @tagName(inst.data.frame_reg.reg), + inst.data.frame_reg.frame, + }), + .frame_addr_reg_mem => try writer.print(".frame_addr_reg_mem {} {s} (tmp {s}), {}", .{ + inst.data.memop_frame_reg.op, + @tagName(inst.data.memop_frame_reg.reg), + @tagName(inst.data.memop_frame_reg.tmp_reg), + inst.data.memop_frame_reg.frame, + }), + .nav_memop => try writer.print(".nav_memop {} {s} (tmp {s}), {} + 0x{x}", .{ + inst.data.memop_nav_reg.op, + @tagName(inst.data.memop_nav_reg.reg), + @tagName(inst.data.memop_nav_reg.tmp_reg), + inst.data.memop_nav_reg.nav.index, + inst.data.memop_nav_reg.nav.off, + }), + .uav_memop => try writer.print(".uav_memop {} {s} (tmp {s}), {} + 0x{x}", .{ + inst.data.memop_uav_reg.op, + @tagName(inst.data.memop_uav_reg.reg), + @tagName(inst.data.memop_uav_reg.tmp_reg), + inst.data.memop_uav_reg.uav.index, + inst.data.memop_uav_reg.uav.off, + }), + .nav_addr_to_reg => try writer.print(".nav_addr_to_reg {} + 0x{x} => {s}", .{ + inst.data.nav_reg.nav.index, + inst.data.nav_reg.nav.off, + @tagName(inst.data.nav_reg.reg), + }), + .uav_addr_to_reg => try writer.print(".uav_addr_to_reg {} + 0x{x} => {s}", .{ + inst.data.uav_reg.uav.index, + inst.data.uav_reg.uav.off, + @tagName(inst.data.uav_reg.reg), + }), + .call => try writer.print(".call nav:{} + 0x{x}", .{ inst.data.nav.index, inst.data.nav.off }), + .spill_int_regs => try writer.print(".spill_int_regs {}", .{inst.data.reg_list.fmt(.int)}), + .spill_float_regs => try writer.print(".spill_float_regs {}", .{inst.data.reg_list.fmt(.float)}), + .restore_int_regs => try writer.print(".restore_int_regs {}", .{inst.data.reg_list.fmt(.int)}), + .restore_float_regs => try writer.print(".restore_float_regs {}", .{inst.data.reg_list.fmt(.float)}), + .dbg_enter_inline_func, .dbg_exit_inline_func => try writer.print(".{s} {}", .{ @tagName(tag), inst.data.func }), + else => try writer.print(".{s}", .{@tagName(tag)}), + } + }, + .inst => |opcode| { + try writer.print("{}", .{Lir.Inst{ .opcode = opcode, .data = inst.data.op }}); + }, + } + } +}; + +comptime { + // Be careful with memory usage when making instruction data larger. + std.debug.assert(@sizeOf(Inst.Data) <= 24); +} + +pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { + mir.instructions.deinit(gpa); + mir.frame_locs.deinit(gpa); + mir.* = undefined; +} + +pub fn emit( + mir: Mir, + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + code: *std.ArrayListUnmanaged(u8), + debug_output: link.File.DebugInfoOutput, +) codegen.CodeGenError!void { + const zcu = pt.zcu; + const comp = zcu.comp; + const gpa = comp.gpa; + const func = zcu.funcInfo(func_index); + const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?; + const nav = func.owner_nav; + const mod = zcu.navFileScope(nav).mod.?; + + var emitter: Emit = .{ + .lower = .{ + .pt = pt, + .link_file = lf, + .target = &mod.resolved_target.result, + .allocator = gpa, + .mir = mir, + .cc = fn_info.cc, + .src_loc = src_loc, + .output_mode = comp.config.output_mode, + .link_mode = comp.config.link_mode, + .pic = mod.pic, + }, + .pt = pt, + .bin_file = lf, + .owner_nav = nav, + .atom_index = sym: { + if (lf.cast(.elf)) |ef| break :sym try ef.zigObjectPtr().?.getOrCreateMetadataForNav(zcu, nav); + unreachable; + }, + .debug_output = debug_output, + .code = code, + .prev_di_loc = .{ + .line = func.lbrace_line, + .column = func.lbrace_column, + .is_stmt = switch (debug_output) { + .dwarf => |dwarf| dwarf.dwarf.debug_line.header.default_is_stmt, + .plan9 => undefined, + .none => undefined, + }, + }, + .prev_di_pc = 0, + }; + emitter.emitMir() catch |err| switch (err) { + error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, emitter.lower.err_msg.?), + else => |e| return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(e)}), + }; +} + +pub const FrameLoc = struct { + base: Register, + offset: i32, + + pub fn format( + loc: FrameLoc, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + if (loc.offset >= 0) { + try writer.print("{s} + 0x{x}", .{ @tagName(loc.base), loc.offset }); + } else { + try writer.print("{s} - 0x{x}", .{ @tagName(loc.base), -loc.offset }); + } + } +}; + +/// Used in conjunction with payload to transfer a list of used registers in a compact manner. +pub const RegisterList = struct { + bitset: BitSet, + + const BitSet = std.bit_set.StaticBitSet(32); + const Self = @This(); + + pub const empty: Self = .{ .bitset = .initEmpty() }; + + pub fn getRegFromIndex(rc: Register.Class, reg: usize) Register { + return .fromClass(rc, @intCast(reg)); + } + + pub fn push(self: *Self, reg: Register) void { + self.bitset.set(reg.enc()); + } + + pub fn isSet(self: Self, reg: Register) bool { + return self.bitset.isSet(reg.enc()); + } + + pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) { + return self.bitset.iterator(options); + } + + pub fn count(self: Self) usize { + return self.bitset.count(); + } + + const FormatData = struct { + regs: *const RegisterList, + rc: Register.Class, + }; + fn format2( + data: FormatData, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + var iter = data.regs.iterator(.{}); + var reg_i: usize = 0; + while (iter.next()) |reg| { + if (reg_i != 0) try writer.writeAll(" "); + reg_i += 1; + try writer.writeAll(@tagName(Register.fromClass(data.rc, @intCast(reg)))); + } + } + fn fmt(regs: *const RegisterList, rc: Register.Class) std.fmt.Formatter(format2) { + return .{ .data = .{ .regs = regs, .rc = rc } }; + } +}; + +pub const BranchCondition = union(enum) { + none, + eq: struct { Register, Register }, + ne: struct { Register, Register }, + le: struct { Register, Register }, + gt: struct { Register, Register }, + leu: struct { Register, Register }, + gtu: struct { Register, Register }, + + pub const Tag = std.meta.Tag(BranchCondition); + + pub fn format( + inst: BranchCondition, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (inst) { + .none => try writer.print("(unconditionally)", .{}), + inline .eq, .ne, .le, .gt, .leu, .gtu => |regs| { + const op = switch (inst) { + .eq => "==", + .ne => "!=", + .le => "<=", + .gt => ">", + .leu => "<= (unsigned)", + .gtu => "> (unsigned)", + else => unreachable, + }; + try writer.print("{s} {s} {s}", .{ @tagName(regs[0]), op, @tagName(regs[1]) }); + }, + } + } + + pub fn compare(tag: Tag, lhs: Register, rhs: Register) BranchCondition { + return switch (tag) { + .none => .none, + .eq => .{ .eq = .{ lhs, rhs } }, + .ne => .{ .ne = .{ lhs, rhs } }, + .le => .{ .le = .{ lhs, rhs } }, + .gt => .{ .gt = .{ lhs, rhs } }, + .leu => .{ .leu = .{ lhs, rhs } }, + .gtu => .{ .gtu = .{ lhs, rhs } }, + }; + } +}; diff --git a/src/arch/loongarch/abi.zig b/src/arch/loongarch/abi.zig new file mode 100644 index 000000000000..9c38f2f96145 --- /dev/null +++ b/src/arch/loongarch/abi.zig @@ -0,0 +1,494 @@ +//! ABI related stuff for LoongArch64. +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +const bits = @import("bits.zig"); +const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; +const Type = @import("../../Type.zig"); +const Zcu = @import("../../Zcu.zig"); +const Air = @import("../../Air.zig"); +const InternPool = @import("../../InternPool.zig"); + +const log = std.log.scoped(.loongarch_abi); + +pub const RegisterClass = enum { + /// Basic integer registers + int, + /// FP/LSX/LASX registers + floating, +}; + +pub const CallInfo = struct { + /// Parameters. + params: []CCValue, + /// Return value. + return_value: CCValue, + + /// Size of the call frame in bytes. + frame_size: u32, + /// Alignment of the call frame. + frame_align: InternPool.Alignment, + /// Register containing pointer to the error return trace struct. + err_ret_trace_reg: ?Register, + + pub fn deinit(self: *CallInfo, allocator: Allocator) void { + allocator.free(self.params); + self.* = undefined; + } +}; + +pub const CCValue = union(enum) { + /// No runtime bits. + none, + /// A value stored in a register. + register: Register, + /// A value stored in two registers. + register_pair: [2]Register, + /// A value stored in three registers. + register_triple: [3]Register, + /// A value stored in four registers. + register_quadruple: [4]Register, + /// A value stored in the call frame with given offset. + frame: i32, + /// A value whose lower-ordered bits are in a register and the others are in the call frame. + split: struct { reg: Register, frame_off: i32 }, + /// A value passed as a pointer in a register. + /// Caller allocates memory and sets the register to the pointer. + ref_register: Register, + /// A value passed as a pointer in the call frame. + /// Caller allocates memory and sets the frame address to the pointer. + ref_frame: i32, + + pub fn getRegs(ccv: *const CCValue) []const Register { + return switch (ccv.*) { + inline .register, .ref_register => |*reg| reg[0..1], + inline .register_pair, .register_triple, .register_quadruple => |*regs| regs, + .split => |*split| (&split.reg)[0..1], + else => &.{}, + }; + } + + pub fn format( + ccv: CCValue, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (ccv) { + .none => try writer.print("({s})", .{@tagName(ccv)}), + .register => |pl| try writer.print("{s}", .{@tagName(pl)}), + .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), + .register_triple => |pl| try writer.print("{s}:{s}:{s}", .{ @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), + .register_quadruple => |pl| try writer.print("{s}:{s}:{s}:{s}", .{ @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), + .frame => |pl| try writer.print("[frame + 0x{x}]", .{pl}), + .split => |pl| try writer.print("{{{s}, (frame + 0x{x})}}", .{ @tagName(pl.reg), pl.frame_off }), + .ref_register => |pl| try writer.print("byref:{s}", .{@tagName(pl)}), + .ref_frame => |pl| try writer.print("byref:[frame + 0x{x}]", .{pl}), + } + } + + pub fn prependReg(ccv: CCValue, reg: Register) ?CCValue { + return switch (ccv) { + .none => .{ .register = reg }, + .register => |pl| .{ .register_pair = .{ reg, pl } }, + .register_pair => |pl| .{ .register_triple = .{ reg, pl[0], pl[1] } }, + .register_triple => |pl| .{ .register_quadruple = .{ reg, pl[0], pl[1], pl[2] } }, + .register_quadruple => null, + .frame => |pl| .{ .split = .{ .reg = reg, .frame_off = pl } }, + .ref_register, .ref_frame, .split => null, + }; + } +}; + +pub const zigcc = struct { + pub const all_allocatable_regs = Integer.all_allocatable_regs ++ Floating.all_allocatable_regs; + pub const all_static = Integer.static_regs ++ Floating.static_regs; + pub const all_temporary = Integer.temporary_regs ++ Floating.temporary_regs; + + pub const Integer = struct { + pub const all_allocatable_regs = function_arg_regs ++ temporary_regs ++ static_regs ++ [_]Register{.ra}; + + pub const function_arg_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r9, .r10, .r11 }; + pub const function_ret_regs = function_arg_regs; + pub const temporary_regs = [_]Register{ .r12, .r13, .r14, .r15, .r16, .r17, .r18, .r19, .r20 }; + pub const static_regs = [_]Register{ .r22, .r23, .r24, .r25, .r26, .r27, .r28, .r29, .r30, .r31 }; + }; + + pub const Floating = struct { + pub const all_allocatable_regs = function_arg_regs ++ temporary_regs ++ static_regs; + + pub const function_arg_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 }; + pub const function_ret_regs = function_arg_regs; + // zig fmt: off + pub const temporary_regs = [_]Register{ + .x8, .x9, .x10, .x11, .x12, .x13, .x14, .x15, + .x16, .x17, .x18, .x19, .x20, .x21, .x22, .x23, + }; + // zig fmt: on + pub const static_regs = [_]Register{ .x24, .x25, .x26, .x27, .x28, .x29, .x30, .x31 }; + }; +}; + +pub const c_abi = struct { + pub const all_allocatable_regs = Integer.all_allocatable_regs ++ Floating.all_allocatable_regs; + pub const all_static = Integer.static_regs ++ Floating.static_regs; + + pub const Integer = struct { + pub const all_allocatable_regs = function_arg_regs ++ temporary_regs ++ static_regs; + + pub const function_arg_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r9, .r10, .r11 }; + pub const function_ret_regs = [_]Register{ .r4, .r5 }; + pub const temporary_regs = [_]Register{ .r12, .r13, .r14, .r15, .r16, .r17, .r18, .r19, .r20 }; + pub const static_regs = [_]Register{ .r22, .r23, .r24, .r25, .r26, .r27, .r28, .r29, .r30, .r31 }; + }; + + pub const Floating = struct { + pub const all_allocatable_regs = function_arg_regs ++ temporary_regs ++ static_regs; + + pub const function_arg_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 }; + pub const function_ret_regs = [_]Register{ .x0, .x1 }; + // zig fmt: off + pub const temporary_regs = [_]Register{ + .x8, .x9, .x10, .x11, .x12, .x13, .x14, .x15, + .x16, .x17, .x18, .x19, .x20, .x21, .x22, .x23, + }; + // zig fmt: on + pub const static_regs = [_]Register{ .x24, .x25, .x26, .x27, .x28, .x29, .x30, .x31 }; + }; +}; + +pub fn getAbiInfo(comptime cc: std.builtin.CallingConvention.Tag) type { + return switch (cc) { + .auto => zigcc, + .loongarch64_lp64 => c_abi, + else => unreachable, + }; +} + +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &zigcc.all_allocatable_regs); +const RegisterBitSet = RegisterManager.RegisterBitSet; + +pub fn getAllocatableRegSet(rc: Register.Class) RegisterBitSet { + return switch (rc) { + .int => RegisterSets.gp, + .float, .lsx, .lasx => RegisterSets.fp, + .fcc => unreachable, // TODO: CFR ABI? + }; +} + +pub const RegisterSets = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (zigcc.all_allocatable_regs, 0..) |reg, index| if (reg.class() == .int) set.set(index); + break :blk set; + }; + pub const fp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (zigcc.all_allocatable_regs, 0..) |reg, index| if (reg.class() == .float) set.set(index); + break :blk set; + }; +}; + +pub const CCResolver = struct { + pt: Zcu.PerThread, + cc: std.builtin.CallingConvention, + + state: struct { + frame_size: u32 = 0, + frame_align: InternPool.Alignment = .@"16", + + reg_count: struct { + param_gpr: u8 = 0, + ret_gpr: u8 = 0, + param_fpr: u8 = 0, + ret_fpr: u8 = 0, + } = .{}, + } = .{}, + + pub fn resolve(pt: Zcu.PerThread, gpa: Allocator, target: *const std.Target, func: *const InternPool.Key.FuncType) !CallInfo { + _ = target; + var resolver: CCResolver = .{ .pt = pt, .cc = func.cc }; + const return_value = try resolver.resolveType(.fromInterned(func.return_type), .ret, 4); + + const params = try gpa.alloc(CCValue, func.param_types.len); + for (func.param_types.get(&pt.zcu.intern_pool), 0..) |param, i| { + const param_ccv = try resolver.resolveType(.fromInterned(param), .param, 4); + params[i] = param_ccv; + } + + const ci: CallInfo = .{ + .params = params, + .return_value = return_value, + .frame_size = resolver.state.frame_size, + .frame_align = resolver.state.frame_align, + .err_ret_trace_reg = null, + }; + log.debug("{}", .{fmtCallInfo(&resolver, &ci, func)}); + return ci; + } + + const Context = enum { param, ret }; + + fn allocRegs(self: *CCResolver, class: RegisterClass, ctx: Context, count: comptime_int) ?[count]Register { + const count_ptr: *u8, const first_reg: Register, const last_reg: Register = switch (class) { + .int => switch (ctx) { + .param => .{ &self.state.reg_count.param_gpr, .r4, .r11 }, + .ret => .{ &self.state.reg_count.ret_gpr, .r4, .r5 }, + }, + .floating => switch (ctx) { + .param => .{ &self.state.reg_count.param_fpr, .x0, .x7 }, + .ret => .{ &self.state.reg_count.ret_fpr, .x0, .x1 }, + }, + }; + if (count_ptr.* + count <= (@intFromEnum(last_reg) - @intFromEnum(first_reg) + 1)) { + var ret: [count]Register = undefined; + for (0..count, count_ptr.*..) |i, off| { + ret[i] = @enumFromInt(@intFromEnum(first_reg) + off); + } + count_ptr.* += count; + return ret; + } else { + return null; + } + } + + fn allocReg(self: *CCResolver, class: RegisterClass, ctx: Context) ?Register { + return if (self.allocRegs(class, ctx, 1)) |regs| + regs[0] + else + null; + } + + /// Allocates a value on stack, returning offset to the args frame + fn allocStackType(self: *CCResolver, ty: Type) i32 { + const zcu = self.pt.zcu; + const alignment = ty.abiAlignment(zcu); + const size = ty.abiSize(zcu); + return self.allocStack(alignment, size); + } + + /// Allocates a value on stack, returning offset to the args frame + fn allocStack(self: *CCResolver, alignment: InternPool.Alignment, size: u64) i32 { + self.state.frame_align = self.state.frame_align.max(alignment); + const off = alignment.forward(self.state.frame_size); + self.state.frame_size = @intCast(off + size); + return @intCast(off); + } + + fn allocPtr(self: *CCResolver, ctx: Context, use_reg: bool) CCValue { + if (use_reg) if (self.allocReg(.int, ctx)) |reg| { + return .{ .register = reg }; + }; + return .{ .frame = self.allocStackType(Type.usize) }; + } + + fn resolveType(self: *CCResolver, ty: Type, ctx: Context, max_regs: u8) error{CCSelectFailed}!CCValue { + const zcu = self.pt.zcu; + // TODO: implement vector calling convention + // TODO: implement FP calling convention + switch (ty.zigTypeTag(zcu)) { + .pointer => switch (ty.ptrSize(zcu)) { + .one, .many, .c => return self.allocPtr(ctx, max_regs >= 1), + .slice => { + if (max_regs >= 2) if (self.allocRegs(.int, ctx, 2)) |regs| { + return .{ .register_pair = regs }; + }; + return .{ .frame = self.allocStack(.@"8", 8 * 2) }; + }, + }, + .int, .@"enum", .bool => { + const ty_size = ty.abiSize(zcu); + switch (ty_size) { + 1...8 => return self.allocPtr(ctx, max_regs >= 1), + 9...16 => { + if (max_regs >= 2) if (self.allocRegs(.int, ctx, 2)) |regs| { + return .{ .register_pair = regs }; + }; + if (max_regs >= 1) + if (self.allocReg(.int, ctx)) |reg| { + return .{ .split = .{ .reg = reg, .frame_off = self.allocStackType(Type.usize) } }; + }; + }, + 17...24 => if (max_regs >= 3) { + if (self.allocRegs(.int, ctx, 3)) |regs| { + return .{ .register_triple = regs }; + } + }, + 25...32 => if (max_regs >= 4) { + if (self.allocRegs(.int, ctx, 4)) |regs| { + return .{ .register_quadruple = regs }; + } + }, + else => {}, + } + return .{ .frame = self.allocStackType(ty) }; + }, + .void, .noreturn => return .none, + .type => unreachable, + .@"struct", .@"union" => { + // TODO: struct flattening + log.warn("Structure flattenning is not implemented yet. The compiled code may misbehave.", .{}); + + const ty_size = ty.bitSize(zcu); + + if (zcu.typeToStruct(ty)) |struct_ty| { + // Structures with floating-point members + const ip = &zcu.intern_pool; + if (struct_ty.field_types.len == 1) { + const member_ty = Type.fromInterned(struct_ty.field_types.get(ip)[0]); + if (member_ty.isRuntimeFloat() and max_regs >= 1) { + if (self.allocReg(.floating, ctx)) |reg| { + return .{ .register = reg }; + } + } + } else if (struct_ty.field_types.len == 2 and max_regs >= 2) { + const member_ty1 = Type.fromInterned(struct_ty.field_types.get(ip)[0]); + const member_ty2 = Type.fromInterned(struct_ty.field_types.get(ip)[1]); + + if (member_ty1.isRuntimeFloat() and member_ty2.isRuntimeFloat()) { + if (self.allocRegs(.floating, ctx, 2)) |regs| { + return .{ .register_pair = regs }; + } + } else if (member_ty1.isRuntimeFloat() and member_ty2.isInt(zcu)) { + const state = self.state; + if (self.allocReg(.floating, ctx)) |reg1| { + if (self.allocReg(.int, ctx)) |reg2| { + return .{ .register_pair = .{ reg1, reg2 } }; + } + } + self.state = state; + } else if (member_ty1.isInt(zcu) and member_ty2.isRuntimeFloat()) { + const state = self.state; + if (self.allocReg(.int, ctx)) |reg1| { + if (self.allocReg(.floating, ctx)) |reg2| { + return .{ .register_pair = .{ reg1, reg2 } }; + } + } + self.state = state; + } + } + } + + // Structures without floating-point members and unions + switch (ty_size) { + 0 => return .none, + 1...64 => return self.allocPtr(ctx, max_regs >= 1), + 65...128 => { + if (max_regs >= 2) if (self.allocRegs(.int, ctx, 2)) |regs| { + return .{ .register_pair = regs }; + }; + if (max_regs >= 1) if (self.allocReg(.int, ctx)) |reg| { + return .{ .split = .{ .reg = reg, .frame_off = self.allocStackType(Type.usize) } }; + }; + return .{ .frame = self.allocStackType(Type.usize) }; + }, + else => if (max_regs >= 1) { + if (self.allocReg(.int, .param)) |reg| { + return .{ .ref_register = reg }; + } + } else return .{ .ref_frame = self.allocStackType(Type.usize) }, + } + }, + .array => { + const ty_size = ty.abiSize(zcu); + switch (ty_size) { + 0 => return .none, + 1...8 => return self.allocPtr(ctx, max_regs >= 1), + 9...16 => { + if (max_regs >= 2) if (self.allocRegs(.int, ctx, 2)) |regs| { + return .{ .register_pair = regs }; + }; + if (max_regs >= 1) if (self.allocReg(.int, ctx)) |reg| { + return .{ .split = .{ .reg = reg, .frame_off = self.allocStackType(Type.usize) } }; + }; + }, + 17...24 => if (max_regs >= 3) { + if (self.allocRegs(.int, ctx, 3)) |regs| { + return .{ .register_triple = regs }; + } + }, + 25...32 => if (max_regs >= 4) { + if (self.allocRegs(.int, ctx, 4)) |regs| { + return .{ .register_quadruple = regs }; + } + }, + else => {}, + } + return .{ .frame = self.allocStackType(ty) }; + }, + .error_set => return self.allocPtr(ctx, max_regs >= 1), + .error_union => { + const payload_ty = ty.errorUnionPayload(zcu); + const payload_bits = payload_ty.bitSize(zcu); + if (payload_bits == 0) return self.allocPtr(ctx, max_regs >= 1); + }, + .optional => { + const child_ty = ty.optionalChild(zcu); + if (child_ty.isPtrAtRuntime(zcu)) return self.allocPtr(ctx, max_regs >= 1); + if (ty.optionalReprIsPayload(zcu)) return self.resolveType(child_ty, ctx, max_regs); + + // try allocate reg / reg + frame + if (max_regs >= 1) { + if (child_ty.isAbiInt(zcu) and ty.abiSize(zcu) <= 8) { + if (self.allocReg(.int, ctx)) |reg| + return .{ .register = reg }; + } + + const state = self.state; + if (self.allocReg(.int, ctx)) |reg| { + const child_ccv = try self.resolveType(child_ty, ctx, max_regs - 1); + if (child_ccv.prependReg(reg)) |ccv| { + return ccv; + } else { + self.state = state; + } + } + } + + // fallback to frame + if (child_ty.abiSize(zcu) > 8 * 2) { + const frame_off = self.allocStack(.@"8", 8); + return .{ .ref_frame = frame_off }; + } else { + const frame_off = self.allocStack(.@"1", 1); + _ = self.allocStackType(child_ty); + return .{ .frame = frame_off }; + } + }, + else => {}, + } + log.err("Failed to select CC location of {} as {s}", .{ ty.fmt(self.pt), @tagName(ctx) }); + return error.CCSelectFailed; + } +}; + +const FormatCallInfoData = struct { + resolver: *const CCResolver, + ci: *const CallInfo, + func: *const InternPool.Key.FuncType, +}; +fn formatCallInfo( + data: FormatCallInfoData, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) @TypeOf(writer).Error!void { + const pt = data.resolver.pt; + const ip = &pt.zcu.intern_pool; + + try writer.writeAll("Calling convention resolve:\n"); + try writer.print(" - CC: {s}\n", .{@tagName(data.resolver.cc)}); + try writer.print(" - ret: {} ({})\n", .{ data.ci.return_value, Type.fromInterned(data.func.return_type).fmt(pt) }); + + for (data.ci.params, 0..) |param, i| { + const param_ty = Type.fromInterned(data.func.param_types.get(ip)[i]); + try writer.print(" - param {}: {} ({})\n", .{ i + 1, param, param_ty.fmt(pt) }); + } + + try writer.print(" - frame: size: {}, align: {}", .{ data.resolver.state.frame_size, data.resolver.state.frame_align.toByteUnits().? }); +} +fn fmtCallInfo(resolver: *const CCResolver, ci: *const CallInfo, func: *const InternPool.Key.FuncType) std.fmt.Formatter(formatCallInfo) { + return .{ .data = .{ .resolver = resolver, .ci = ci, .func = func } }; +} diff --git a/src/arch/loongarch/bits.zig b/src/arch/loongarch/bits.zig new file mode 100644 index 000000000000..71bc98951916 --- /dev/null +++ b/src/arch/loongarch/bits.zig @@ -0,0 +1,332 @@ +const std = @import("std"); +const Target = std.Target; +const expectEqual = std.testing.expectEqual; + +const InternPool = @import("../../InternPool.zig"); + +pub const Register = enum(u8) { + // zig fmt: off + // integer registers + r0, r1, r2, r3, r4, r5, r6, r7, + r8, r9, r10, r11, r12, r13, r14, r15, + r16, r17, r18, r19, r20, r21, r22, r23, + r24, r25, r26, r27, r28, r29, r30, r31, + + // float-point registers + f0, f1, f2, f3, f4, f5, f6, f7, + f8, f9, f10, f11, f12, f13, f14, f15, + f16, f17, f18, f19, f20, f21, f22, f23, + f24, f25, f26, f27, f28, f29, f30, f31, + + // LSX registers + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, + + // LASX registers + x0, x1, x2, x3, x4, x5, x6, x7, + x8, x9, x10, x11, x12, x13, x14, x15, + x16, x17, x18, x19, x20, x21, x22, x23, + x24, x25, x26, x27, x28, x29, x30, x31, + + // float-point condition code registers + fcc0, fcc1, fcc2, fcc3, fcc4, fcc5, fcc6, fcc7, + // zig fmt: on + + pub const zero = Register.r0; + pub const ra = Register.r1; + pub const tp = Register.r2; + pub const sp = Register.r3; + pub const fp = Register.r22; + pub const t0 = Register.r12; + pub const t1 = Register.r13; + pub const t2 = Register.r14; + pub const t3 = Register.r15; + + pub const Class = enum { + int, + float, + lsx, + lasx, + fcc, + + pub fn byteSize(rc: Class, target: *const Target) usize { + return switch (rc) { + .int => switch (target.cpu.arch) { + .loongarch32 => 4, + .loongarch64 => 8, + else => unreachable, + }, + .float => 8, + .lsx => 16, + .lasx => 32, + .fcc => 1, + }; + } + }; + + pub fn fromClass(cls: Class, reg: u5) Register { + const base: u8 = switch (cls) { + .int => @intFromEnum(Register.r0), + .float => @intFromEnum(Register.f0), + .lsx => @intFromEnum(Register.v0), + .lasx => @intFromEnum(Register.x0), + .fcc => @intFromEnum(Register.fcc0), + }; + return @enumFromInt(base + reg); + } + + pub fn class(reg: Register) Class { + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.r0)...@intFromEnum(Register.r31) => .int, + @intFromEnum(Register.f0)...@intFromEnum(Register.f31) => .float, + @intFromEnum(Register.v0)...@intFromEnum(Register.v31) => .lsx, + @intFromEnum(Register.x0)...@intFromEnum(Register.x31) => .lasx, + @intFromEnum(Register.fcc0)...@intFromEnum(Register.fcc7) => .fcc, + else => unreachable, + }; + } + + pub fn id(reg: Register) u8 { + // LSX and LASX registers are aliased to float-point registers + // because their lower bits are shared + const base: u8 = switch (@intFromEnum(reg)) { + @intFromEnum(Register.r0)...@intFromEnum(Register.r31) => 0, + @intFromEnum(Register.f0)...@intFromEnum(Register.f31) => 32, + @intFromEnum(Register.v0)...@intFromEnum(Register.v31) => 32, + @intFromEnum(Register.x0)...@intFromEnum(Register.x31) => 32, + @intFromEnum(Register.fcc0)...@intFromEnum(Register.fcc7) => 64, + else => unreachable, + }; + return @as(u8, @intCast(reg.enc())) + base; + } + + pub fn enc(reg: Register) u5 { + const base: u8 = switch (@intFromEnum(reg)) { + @intFromEnum(Register.r0)...@intFromEnum(Register.r31) => @intFromEnum(Register.r0), + @intFromEnum(Register.f0)...@intFromEnum(Register.f31) => @intFromEnum(Register.f0), + @intFromEnum(Register.v0)...@intFromEnum(Register.v31) => @intFromEnum(Register.v0), + @intFromEnum(Register.x0)...@intFromEnum(Register.x31) => @intFromEnum(Register.x0), + @intFromEnum(Register.fcc0)...@intFromEnum(Register.fcc7) => @intFromEnum(Register.fcc0), + else => unreachable, + }; + return @intCast(@intFromEnum(reg) - base); + } + + pub fn bitSize(reg: Register) u10 { + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.r0)...@intFromEnum(Register.r31) => 64, + @intFromEnum(Register.f0)...@intFromEnum(Register.f31) => 64, + @intFromEnum(Register.v0)...@intFromEnum(Register.v31) => 128, + @intFromEnum(Register.x0)...@intFromEnum(Register.x31) => 256, + @intFromEnum(Register.fcc0)...@intFromEnum(Register.fcc7) => 1, + else => unreachable, + }; + } + + pub fn dwarfNum(reg: Register) u6 { + return switch (reg.class()) { + .int => @as(u6, reg.id()), + .float => 32 + @as(u6, reg.id()), + // TODO + .lsx, .lasx => unreachable, + .fcc => unreachable, + }; + } + + /// Converts a FP/LSX/LASX register to FP register. + pub fn toFloat(reg: Register) Register { + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.f0)...@intFromEnum(Register.f31) => reg, + @intFromEnum(Register.v0)...@intFromEnum(Register.v31), + @intFromEnum(Register.x0)...@intFromEnum(Register.x31), + => @enumFromInt(@intFromEnum(Register.f0) + reg.enc()), + else => unreachable, + }; + } + + /// Converts a FP/LSX/LASX register to LSX register. + pub fn toLsx(reg: Register) Register { + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.v0)...@intFromEnum(Register.v31) => reg, + @intFromEnum(Register.f0)...@intFromEnum(Register.f31), + @intFromEnum(Register.x0)...@intFromEnum(Register.x31), + => @enumFromInt(@intFromEnum(Register.v0) + reg.enc()), + else => unreachable, + }; + } + + /// Converts a FP/LSX/LASX register to LASX register. + pub fn toLasx(reg: Register) Register { + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.x0)...@intFromEnum(Register.x31) => reg, + @intFromEnum(Register.f0)...@intFromEnum(Register.f31), + @intFromEnum(Register.v0)...@intFromEnum(Register.v31), + => @enumFromInt(@intFromEnum(Register.x0) + reg.enc()), + else => unreachable, + }; + } +}; + +test "register classes" { + try expectEqual(.int, Register.r0.class()); + try expectEqual(.int, Register.r31.class()); + try expectEqual(.float, Register.f0.class()); + try expectEqual(.float, Register.f31.class()); + try expectEqual(.lsx, Register.v0.class()); + try expectEqual(.lsx, Register.v31.class()); + try expectEqual(.lasx, Register.x0.class()); + try expectEqual(.lasx, Register.x31.class()); + try expectEqual(.fcc, Register.fcc0.class()); + try expectEqual(.fcc, Register.fcc7.class()); +} + +test "register id" { + try expectEqual(0, Register.r0.enc()); + try expectEqual(31, Register.r31.enc()); + try expectEqual(0, Register.f0.enc()); + try expectEqual(31, Register.f31.enc()); + try expectEqual(0, Register.v0.enc()); + try expectEqual(31, Register.v31.enc()); + try expectEqual(0, Register.x0.enc()); + try expectEqual(31, Register.x31.enc()); + try expectEqual(0, Register.fcc0.enc()); + try expectEqual(7, Register.fcc7.enc()); +} + +test "register encoding" { + try expectEqual(0, Register.r0.id()); + try expectEqual(31, Register.r31.id()); + try expectEqual(32, Register.f0.id()); + try expectEqual(63, Register.f31.id()); + try expectEqual(32, Register.v0.id()); + try expectEqual(63, Register.v31.id()); + try expectEqual(32, Register.x0.id()); + try expectEqual(63, Register.x31.id()); + try expectEqual(64, Register.fcc0.id()); + try expectEqual(71, Register.fcc7.id()); +} + +test "register decoding" { + try expectEqual(Register.r0, Register.fromClass(.int, 0)); + try expectEqual(Register.r31, Register.fromClass(.int, 31)); + try expectEqual(Register.f0, Register.fromClass(.float, 0)); + try expectEqual(Register.f31, Register.fromClass(.float, 31)); + try expectEqual(Register.v0, Register.fromClass(.lsx, 0)); + try expectEqual(Register.v31, Register.fromClass(.lsx, 31)); + try expectEqual(Register.x0, Register.fromClass(.lasx, 0)); + try expectEqual(Register.x31, Register.fromClass(.lasx, 31)); + try expectEqual(Register.fcc0, Register.fromClass(.fcc, 0)); + try expectEqual(Register.fcc7, Register.fromClass(.fcc, 7)); +} + +pub const FrameIndex = enum(u32) { + /// Refers to the start of the arguments passed to this function + args_frame, + /// Refers to the start of spilled return address. + ret_addr_frame, + /// Refers to the start of spilled integer static registers. + spill_int_frame, + /// Refers to the start of spilled floating-point static registers. + spill_float_frame, + // /// Refers to the base pointer pushed in the prologue and popped in the epilogue. + // base_ptr, + /// Refers to the start of the call frame for arguments passed to called functions + call_frame, + /// Refers to the entire stack frame. + stack_frame, + /// Other indices are used for local variable stack slots + _, + + pub const named_count = @typeInfo(FrameIndex).@"enum".fields.len; + + pub fn isNamed(fi: FrameIndex) bool { + return @intFromEnum(fi) < named_count; + } + + pub fn format( + fi: FrameIndex, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + try writer.writeAll("FrameIndex"); + if (fi.isNamed()) { + try writer.writeByte('.'); + try writer.writeAll(@tagName(fi)); + } else { + try writer.writeByte('('); + try std.fmt.formatType(@intFromEnum(fi), fmt, options, writer, 0); + try writer.writeByte(')'); + } + } +}; + +pub const FrameAddr = struct { + index: FrameIndex, + off: i32 = 0, + + pub fn format( + addr: FrameAddr, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + if (addr.off >= 0) { + try writer.print("{} + 0x{x}", .{ addr.index, addr.off }); + } else { + try writer.print("{} - 0x{x}", .{ addr.index, -addr.off }); + } + } +}; + +pub const RegisterOffset = struct { reg: Register, off: i32 = 0 }; + +pub const RegisterFrame = struct { reg: Register, frame: FrameAddr }; + +pub const NavOffset = struct { index: InternPool.Nav.Index, off: i32 = 0 }; + +pub const UavOffset = struct { index: InternPool.Key.Ptr.BaseAddr.Uav, off: i32 = 0 }; + +pub const Memory = struct { + pub const Size = enum(u4) { + /// Byte, 1 byte + byte, + /// Half word, 2 bytes + hword, + /// Word, 4 bytes + word, + /// Double word, 8 Bytes + dword, + + pub fn fromByteSize(size: u64) Size { + return switch (size) { + 1...1 => .byte, + 2...2 => .hword, + 3...4 => .word, + 5...8 => .dword, + else => std.debug.panic("fromByteSize {}", .{size}), + }; + } + + pub fn fromBitSize(bit_size: u64) Size { + return switch (bit_size) { + 8 => .byte, + 16 => .hword, + 32 => .word, + 64 => .dword, + else => unreachable, + }; + } + + pub fn bitSize(s: Size) u64 { + return switch (s) { + .byte => 8, + .hword => 16, + .word => 32, + .dword => 64, + }; + } + }; +}; diff --git a/src/arch/loongarch/encoding.zig b/src/arch/loongarch/encoding.zig new file mode 100644 index 000000000000..7fc139ebeb5b --- /dev/null +++ b/src/arch/loongarch/encoding.zig @@ -0,0 +1,2864 @@ +// This file is auto-generated by tools/gen_loongarch_encoding.zig + +const Register = @import("bits.zig").Register; + +pub const OpCode = enum { + add_d, + add_w, + addi_d, + addi_w, + addu16i_d, + amadd_b, + amadd_d, + amadd_h, + amadd_w, + amadd_db_b, + amadd_db_d, + amadd_db_h, + amadd_db_w, + amand_d, + amand_w, + amand_db_d, + amand_db_w, + amcas_b, + amcas_d, + amcas_h, + amcas_w, + amcas_db_b, + amcas_db_d, + amcas_db_h, + amcas_db_w, + ammax_d, + ammax_du, + ammax_w, + ammax_wu, + ammax_db_d, + ammax_db_du, + ammax_db_w, + ammax_db_wu, + ammin_d, + ammin_du, + ammin_w, + ammin_wu, + ammin_db_d, + ammin_db_du, + ammin_db_w, + ammin_db_wu, + amor_d, + amor_w, + amor_db_d, + amor_db_w, + amswap_b, + amswap_d, + amswap_h, + amswap_w, + amswap_db_b, + amswap_db_d, + amswap_db_h, + amswap_db_w, + amxor_d, + amxor_w, + amxor_db_d, + amxor_db_w, + @"and", + andi, + andn, + asrtgt, + asrtle, + b, + bceqz, + bcnez, + beq, + beqz, + bgt, + bgtu, + bl, + ble, + bleu, + bne, + bnez, + @"break", + bstrins_d, + bstrins_w, + bstrpick_d, + bstrpick_w, + cacop, + catpick_d, + catpick_w, + clo_d, + clo_w, + clz_d, + clz_w, + cpucfg, + crc_w_b_w, + crc_w_d_w, + crc_w_h_w, + crc_w_w_w, + crcc_w_b_w, + crcc_w_d_w, + crcc_w_h_w, + crcc_w_w_w, + csrxchg, + cto_d, + cto_w, + ctz_d, + ctz_w, + cu32i_d, + cu52i_d, + dbar, + dbgcall, + div_d, + div_du, + div_w, + div_wu, + eret, + fabs_d, + fabs_s, + fadd_d, + fadd_s, + fclass_d, + fclass_s, + fcmp_caf_d, + fcmp_caf_s, + fcmp_ceq_d, + fcmp_ceq_s, + fcmp_cle_d, + fcmp_cle_s, + fcmp_clt_d, + fcmp_clt_s, + fcmp_cne_d, + fcmp_cne_s, + fcmp_cor_d, + fcmp_cor_s, + fcmp_cueq_d, + fcmp_cueq_s, + fcmp_cule_d, + fcmp_cule_s, + fcmp_cult_d, + fcmp_cult_s, + fcmp_cun_d, + fcmp_cun_s, + fcmp_cune_d, + fcmp_cune_s, + fcmp_saf_d, + fcmp_saf_s, + fcmp_seq_d, + fcmp_seq_s, + fcmp_sle_d, + fcmp_sle_s, + fcmp_slt_d, + fcmp_slt_s, + fcmp_sne_d, + fcmp_sne_s, + fcmp_sor_d, + fcmp_sor_s, + fcmp_sueq_d, + fcmp_sueq_s, + fcmp_sule_d, + fcmp_sule_s, + fcmp_sult_d, + fcmp_sult_s, + fcmp_sun_d, + fcmp_sun_s, + fcmp_sune_d, + fcmp_sune_s, + fcopysign_d, + fcopysign_s, + fcsrrd, + fcsrwr, + fcvt_d_s, + fcvt_s_d, + fdiv_d, + fdiv_s, + ffint_d_l, + ffint_d_w, + ffint_s_l, + ffint_s_w, + fld_d, + fld_s, + fldgt_d, + fldgt_s, + fldle_d, + fldle_s, + fldx_d, + fldx_s, + flogb_d, + flogb_s, + fmadd_d, + fmadd_s, + fmax_d, + fmax_s, + fmaxa_d, + fmaxa_s, + fmin_d, + fmin_s, + fmina_d, + fmina_s, + fmov_d, + fmov_s, + fmsub_d, + fmsub_s, + fmul_d, + fmul_s, + fneg_d, + fneg_s, + fnmadd_d, + fnmadd_s, + fnmsub_d, + fnmsub_s, + frecip_d, + frecip_s, + frecipe_d, + frecipe_s, + frint_d, + frint_s, + frsqrt_d, + frsqrt_s, + frsqrte_d, + frsqrte_s, + fscaleb_d, + fscaleb_s, + fsel, + fsqrt_d, + fsqrt_s, + fst_d, + fst_s, + fstgt_d, + fstgt_s, + fstle_d, + fstle_s, + fstx_d, + fstx_s, + fsub_d, + fsub_s, + ftint_l_d, + ftint_l_s, + ftint_w_d, + ftint_w_s, + ftintrm_l_d, + ftintrm_l_s, + ftintrm_w_d, + ftintrm_w_s, + ftintrne_l_d, + ftintrne_l_s, + ftintrne_w_d, + ftintrne_w_s, + ftintrp_l_d, + ftintrp_l_s, + ftintrp_w_d, + ftintrp_w_s, + ftintrz_l_d, + ftintrz_l_s, + ftintrz_w_d, + ftintrz_w_s, + ibar, + idle, + iocsrrd_b, + iocsrrd_d, + iocsrrd_h, + iocsrrd_w, + iocsrwr_b, + iocsrwr_d, + iocsrwr_h, + iocsrwr_w, + jirl, + ld_b, + ld_bu, + ld_d, + ld_h, + ld_hu, + ld_w, + ld_wu, + lddir, + ldgt_b, + ldgt_d, + ldgt_h, + ldgt_w, + ldle_b, + ldle_d, + ldle_h, + ldle_w, + ldox4_d, + ldox4_w, + ldpte, + ldx_b, + ldx_bu, + ldx_d, + ldx_h, + ldx_hu, + ldx_w, + ldx_wu, + ll_d, + ll_w, + llacq_d, + llacq_w, + lu12i_w, + maskeqz, + masknez, + mod_d, + mod_du, + mod_w, + mod_wu, + movfcc2fr, + movfcc2gr, + movfr2fcc, + movfr2gr_d, + movfr2gr_s, + movfrh2gr_s, + movgr2fcc, + movgr2fr_d, + movgr2fr_w, + movgr2frh_w, + mul_d, + mul_w, + mulh_d, + mulh_du, + mulh_w, + mulh_wu, + mulw_d_w, + mulw_d_wu, + nor, + @"or", + ori, + orn, + pcaddu12i, + pcaddu18i, + pcaddu2i, + pcalau12i, + preld, + preldx, + rdtime_d, + rdtimeh_w, + rdtimel_w, + revb_2h, + revb_2w, + revb_4h, + revb_d, + revbit_4b, + revbit_8b, + revbit_d, + revbit_w, + revh_2w, + revh_d, + rotr_d, + rotr_w, + rotri_d, + rotri_w, + sc_d, + sc_q, + sc_w, + screl_d, + screl_w, + sext_b, + sext_h, + sladd_d, + sladd_w, + sladd_wu, + sll_d, + sll_w, + slli_d, + slli_w, + slt, + slti, + sltu, + sltui, + sra_d, + sra_w, + srai_d, + srai_w, + srl_d, + srl_w, + srli_d, + srli_w, + st_b, + st_d, + st_h, + st_w, + stgt_b, + stgt_d, + stgt_h, + stgt_w, + stle_b, + stle_d, + stle_h, + stle_w, + stox4_d, + stox4_w, + stx_b, + stx_d, + stx_h, + stx_w, + sub_d, + sub_w, + syscall, + tlbclr, + tlbfill, + tlbflush, + tlbinv, + tlbrd, + tlbsrch, + tlbwr, + xor, + xori, + xxx_unknown_1, + + pub fn enc(opcode: OpCode) u32 { + return switch (opcode) { + .add_d => 0x00108000, + .add_w => 0x00100000, + .addi_d => 0x02c00000, + .addi_w => 0x02800000, + .addu16i_d => 0x10000000, + .amadd_b => 0x385d0000, + .amadd_d => 0x38618000, + .amadd_h => 0x385d8000, + .amadd_w => 0x38610000, + .amadd_db_b => 0x385f0000, + .amadd_db_d => 0x386a8000, + .amadd_db_h => 0x385f8000, + .amadd_db_w => 0x386a0000, + .amand_d => 0x38628000, + .amand_w => 0x38620000, + .amand_db_d => 0x386b8000, + .amand_db_w => 0x386b0000, + .amcas_b => 0x38580000, + .amcas_d => 0x38598000, + .amcas_h => 0x38588000, + .amcas_w => 0x38590000, + .amcas_db_b => 0x385a0000, + .amcas_db_d => 0x385b8000, + .amcas_db_h => 0x385a8000, + .amcas_db_w => 0x385b0000, + .ammax_d => 0x38658000, + .ammax_du => 0x38678000, + .ammax_w => 0x38650000, + .ammax_wu => 0x38670000, + .ammax_db_d => 0x386e8000, + .ammax_db_du => 0x38708000, + .ammax_db_w => 0x386e0000, + .ammax_db_wu => 0x38700000, + .ammin_d => 0x38668000, + .ammin_du => 0x38688000, + .ammin_w => 0x38660000, + .ammin_wu => 0x38680000, + .ammin_db_d => 0x386f8000, + .ammin_db_du => 0x38718000, + .ammin_db_w => 0x386f0000, + .ammin_db_wu => 0x38710000, + .amor_d => 0x38638000, + .amor_w => 0x38630000, + .amor_db_d => 0x386c8000, + .amor_db_w => 0x386c0000, + .amswap_b => 0x385c0000, + .amswap_d => 0x38608000, + .amswap_h => 0x385c8000, + .amswap_w => 0x38600000, + .amswap_db_b => 0x385e0000, + .amswap_db_d => 0x38698000, + .amswap_db_h => 0x385e8000, + .amswap_db_w => 0x38690000, + .amxor_d => 0x38648000, + .amxor_w => 0x38640000, + .amxor_db_d => 0x386d8000, + .amxor_db_w => 0x386d0000, + .@"and" => 0x00148000, + .andi => 0x03400000, + .andn => 0x00168000, + .asrtgt => 0x00018000, + .asrtle => 0x00010000, + .b => 0x50000000, + .bceqz => 0x48000000, + .bcnez => 0x48000100, + .beq => 0x58000000, + .beqz => 0x40000000, + .bgt => 0x60000000, + .bgtu => 0x68000000, + .bl => 0x54000000, + .ble => 0x64000000, + .bleu => 0x6c000000, + .bne => 0x5c000000, + .bnez => 0x44000000, + .@"break" => 0x002a0000, + .bstrins_d => 0x00800000, + .bstrins_w => 0x00600000, + .bstrpick_d => 0x00c00000, + .bstrpick_w => 0x00608000, + .cacop => 0x06000000, + .catpick_d => 0x000c0000, + .catpick_w => 0x00080000, + .clo_d => 0x00002000, + .clo_w => 0x00001000, + .clz_d => 0x00002400, + .clz_w => 0x00001400, + .cpucfg => 0x00006c00, + .crc_w_b_w => 0x00240000, + .crc_w_d_w => 0x00258000, + .crc_w_h_w => 0x00248000, + .crc_w_w_w => 0x00250000, + .crcc_w_b_w => 0x00260000, + .crcc_w_d_w => 0x00278000, + .crcc_w_h_w => 0x00268000, + .crcc_w_w_w => 0x00270000, + .csrxchg => 0x04000000, + .cto_d => 0x00002800, + .cto_w => 0x00001800, + .ctz_d => 0x00002c00, + .ctz_w => 0x00001c00, + .cu32i_d => 0x16000000, + .cu52i_d => 0x03000000, + .dbar => 0x38720000, + .dbgcall => 0x002a8000, + .div_d => 0x00220000, + .div_du => 0x00230000, + .div_w => 0x00200000, + .div_wu => 0x00210000, + .eret => 0x06483800, + .fabs_d => 0x01140800, + .fabs_s => 0x01140400, + .fadd_d => 0x01010000, + .fadd_s => 0x01008000, + .fclass_d => 0x01143800, + .fclass_s => 0x01143400, + .fcmp_caf_d => 0x0c200000, + .fcmp_caf_s => 0x0c100000, + .fcmp_ceq_d => 0x0c220000, + .fcmp_ceq_s => 0x0c120000, + .fcmp_cle_d => 0x0c230000, + .fcmp_cle_s => 0x0c130000, + .fcmp_clt_d => 0x0c210000, + .fcmp_clt_s => 0x0c110000, + .fcmp_cne_d => 0x0c280000, + .fcmp_cne_s => 0x0c180000, + .fcmp_cor_d => 0x0c2a0000, + .fcmp_cor_s => 0x0c1a0000, + .fcmp_cueq_d => 0x0c260000, + .fcmp_cueq_s => 0x0c160000, + .fcmp_cule_d => 0x0c270000, + .fcmp_cule_s => 0x0c170000, + .fcmp_cult_d => 0x0c250000, + .fcmp_cult_s => 0x0c150000, + .fcmp_cun_d => 0x0c240000, + .fcmp_cun_s => 0x0c140000, + .fcmp_cune_d => 0x0c2c0000, + .fcmp_cune_s => 0x0c1c0000, + .fcmp_saf_d => 0x0c208000, + .fcmp_saf_s => 0x0c108000, + .fcmp_seq_d => 0x0c228000, + .fcmp_seq_s => 0x0c128000, + .fcmp_sle_d => 0x0c238000, + .fcmp_sle_s => 0x0c138000, + .fcmp_slt_d => 0x0c218000, + .fcmp_slt_s => 0x0c118000, + .fcmp_sne_d => 0x0c288000, + .fcmp_sne_s => 0x0c188000, + .fcmp_sor_d => 0x0c2a8000, + .fcmp_sor_s => 0x0c1a8000, + .fcmp_sueq_d => 0x0c268000, + .fcmp_sueq_s => 0x0c168000, + .fcmp_sule_d => 0x0c278000, + .fcmp_sule_s => 0x0c178000, + .fcmp_sult_d => 0x0c258000, + .fcmp_sult_s => 0x0c158000, + .fcmp_sun_d => 0x0c248000, + .fcmp_sun_s => 0x0c148000, + .fcmp_sune_d => 0x0c2c8000, + .fcmp_sune_s => 0x0c1c8000, + .fcopysign_d => 0x01130000, + .fcopysign_s => 0x01128000, + .fcsrrd => 0x0114c800, + .fcsrwr => 0x0114c000, + .fcvt_d_s => 0x01192400, + .fcvt_s_d => 0x01191800, + .fdiv_d => 0x01070000, + .fdiv_s => 0x01068000, + .ffint_d_l => 0x011d2800, + .ffint_d_w => 0x011d2000, + .ffint_s_l => 0x011d1800, + .ffint_s_w => 0x011d1000, + .fld_d => 0x2b800000, + .fld_s => 0x2b000000, + .fldgt_d => 0x38748000, + .fldgt_s => 0x38740000, + .fldle_d => 0x38758000, + .fldle_s => 0x38750000, + .fldx_d => 0x38340000, + .fldx_s => 0x38300000, + .flogb_d => 0x01142800, + .flogb_s => 0x01142400, + .fmadd_d => 0x08200000, + .fmadd_s => 0x08100000, + .fmax_d => 0x01090000, + .fmax_s => 0x01088000, + .fmaxa_d => 0x010d0000, + .fmaxa_s => 0x010c8000, + .fmin_d => 0x010b0000, + .fmin_s => 0x010a8000, + .fmina_d => 0x010f0000, + .fmina_s => 0x010e8000, + .fmov_d => 0x01149800, + .fmov_s => 0x01149400, + .fmsub_d => 0x08600000, + .fmsub_s => 0x08500000, + .fmul_d => 0x01050000, + .fmul_s => 0x01048000, + .fneg_d => 0x01141800, + .fneg_s => 0x01141400, + .fnmadd_d => 0x08a00000, + .fnmadd_s => 0x08900000, + .fnmsub_d => 0x08e00000, + .fnmsub_s => 0x08d00000, + .frecip_d => 0x01145800, + .frecip_s => 0x01145400, + .frecipe_d => 0x01147800, + .frecipe_s => 0x01147400, + .frint_d => 0x011e4800, + .frint_s => 0x011e4400, + .frsqrt_d => 0x01146800, + .frsqrt_s => 0x01146400, + .frsqrte_d => 0x01148800, + .frsqrte_s => 0x01148400, + .fscaleb_d => 0x01110000, + .fscaleb_s => 0x01108000, + .fsel => 0x0d000000, + .fsqrt_d => 0x01144800, + .fsqrt_s => 0x01144400, + .fst_d => 0x2bc00000, + .fst_s => 0x2b400000, + .fstgt_d => 0x38768000, + .fstgt_s => 0x38760000, + .fstle_d => 0x38778000, + .fstle_s => 0x38770000, + .fstx_d => 0x383c0000, + .fstx_s => 0x38380000, + .fsub_d => 0x01030000, + .fsub_s => 0x01028000, + .ftint_l_d => 0x011b2800, + .ftint_l_s => 0x011b2400, + .ftint_w_d => 0x011b0800, + .ftint_w_s => 0x011b0400, + .ftintrm_l_d => 0x011a2800, + .ftintrm_l_s => 0x011a2400, + .ftintrm_w_d => 0x011a0800, + .ftintrm_w_s => 0x011a0400, + .ftintrne_l_d => 0x011ae800, + .ftintrne_l_s => 0x011ae400, + .ftintrne_w_d => 0x011ac800, + .ftintrne_w_s => 0x011ac400, + .ftintrp_l_d => 0x011a6800, + .ftintrp_l_s => 0x011a6400, + .ftintrp_w_d => 0x011a4800, + .ftintrp_w_s => 0x011a4400, + .ftintrz_l_d => 0x011aa800, + .ftintrz_l_s => 0x011aa400, + .ftintrz_w_d => 0x011a8800, + .ftintrz_w_s => 0x011a8400, + .ibar => 0x38728000, + .idle => 0x06488000, + .iocsrrd_b => 0x06480000, + .iocsrrd_d => 0x06480c00, + .iocsrrd_h => 0x06480400, + .iocsrrd_w => 0x06480800, + .iocsrwr_b => 0x06481000, + .iocsrwr_d => 0x06481c00, + .iocsrwr_h => 0x06481400, + .iocsrwr_w => 0x06481800, + .jirl => 0x4c000000, + .ld_b => 0x28000000, + .ld_bu => 0x2a000000, + .ld_d => 0x28c00000, + .ld_h => 0x28400000, + .ld_hu => 0x2a400000, + .ld_w => 0x28800000, + .ld_wu => 0x2a800000, + .lddir => 0x06400000, + .ldgt_b => 0x38780000, + .ldgt_d => 0x38798000, + .ldgt_h => 0x38788000, + .ldgt_w => 0x38790000, + .ldle_b => 0x387a0000, + .ldle_d => 0x387b8000, + .ldle_h => 0x387a8000, + .ldle_w => 0x387b0000, + .ldox4_d => 0x26000000, + .ldox4_w => 0x24000000, + .ldpte => 0x06440000, + .ldx_b => 0x38000000, + .ldx_bu => 0x38200000, + .ldx_d => 0x380c0000, + .ldx_h => 0x38040000, + .ldx_hu => 0x38240000, + .ldx_w => 0x38080000, + .ldx_wu => 0x38280000, + .ll_d => 0x22000000, + .ll_w => 0x20000000, + .llacq_d => 0x38578800, + .llacq_w => 0x38578000, + .lu12i_w => 0x14000000, + .maskeqz => 0x00130000, + .masknez => 0x00138000, + .mod_d => 0x00228000, + .mod_du => 0x00238000, + .mod_w => 0x00208000, + .mod_wu => 0x00218000, + .movfcc2fr => 0x0114d400, + .movfcc2gr => 0x0114dc00, + .movfr2fcc => 0x0114d000, + .movfr2gr_d => 0x0114b800, + .movfr2gr_s => 0x0114b400, + .movfrh2gr_s => 0x0114bc00, + .movgr2fcc => 0x0114d800, + .movgr2fr_d => 0x0114a800, + .movgr2fr_w => 0x0114a400, + .movgr2frh_w => 0x0114ac00, + .mul_d => 0x001d8000, + .mul_w => 0x001c0000, + .mulh_d => 0x001e0000, + .mulh_du => 0x001e8000, + .mulh_w => 0x001c8000, + .mulh_wu => 0x001d0000, + .mulw_d_w => 0x001f0000, + .mulw_d_wu => 0x001f8000, + .nor => 0x00140000, + .@"or" => 0x00150000, + .ori => 0x03800000, + .orn => 0x00160000, + .pcaddu12i => 0x1c000000, + .pcaddu18i => 0x1e000000, + .pcaddu2i => 0x18000000, + .pcalau12i => 0x1a000000, + .preld => 0x2ac00000, + .preldx => 0x382c0000, + .rdtime_d => 0x00006800, + .rdtimeh_w => 0x00006400, + .rdtimel_w => 0x00006000, + .revb_2h => 0x00003000, + .revb_2w => 0x00003800, + .revb_4h => 0x00003400, + .revb_d => 0x00003c00, + .revbit_4b => 0x00004800, + .revbit_8b => 0x00004c00, + .revbit_d => 0x00005400, + .revbit_w => 0x00005000, + .revh_2w => 0x00004000, + .revh_d => 0x00004400, + .rotr_d => 0x001b8000, + .rotr_w => 0x001b0000, + .rotri_d => 0x004d0000, + .rotri_w => 0x004c8000, + .sc_d => 0x23000000, + .sc_q => 0x38570000, + .sc_w => 0x21000000, + .screl_d => 0x38578c00, + .screl_w => 0x38578400, + .sext_b => 0x00005c00, + .sext_h => 0x00005800, + .sladd_d => 0x002c0000, + .sladd_w => 0x00040000, + .sladd_wu => 0x00060000, + .sll_d => 0x00188000, + .sll_w => 0x00170000, + .slli_d => 0x00410000, + .slli_w => 0x00408000, + .slt => 0x00120000, + .slti => 0x02000000, + .sltu => 0x00128000, + .sltui => 0x02400000, + .sra_d => 0x00198000, + .sra_w => 0x00180000, + .srai_d => 0x00490000, + .srai_w => 0x00488000, + .srl_d => 0x00190000, + .srl_w => 0x00178000, + .srli_d => 0x00450000, + .srli_w => 0x00448000, + .st_b => 0x29000000, + .st_d => 0x29c00000, + .st_h => 0x29400000, + .st_w => 0x29800000, + .stgt_b => 0x387c0000, + .stgt_d => 0x387d8000, + .stgt_h => 0x387c8000, + .stgt_w => 0x387d0000, + .stle_b => 0x387e0000, + .stle_d => 0x387f8000, + .stle_h => 0x387e8000, + .stle_w => 0x387f0000, + .stox4_d => 0x27000000, + .stox4_w => 0x25000000, + .stx_b => 0x38100000, + .stx_d => 0x381c0000, + .stx_h => 0x38140000, + .stx_w => 0x38180000, + .sub_d => 0x00118000, + .sub_w => 0x00110000, + .syscall => 0x002b0000, + .tlbclr => 0x06482000, + .tlbfill => 0x06483400, + .tlbflush => 0x06482400, + .tlbinv => 0x06498000, + .tlbrd => 0x06482c00, + .tlbsrch => 0x06482800, + .tlbwr => 0x06483000, + .xor => 0x00158000, + .xori => 0x03c00000, + .xxx_unknown_1 => 0x06493000, + }; + } +}; + +pub const Data = union(enum) { + DJUk8: struct { Register, Register, u8 }, + DJUk6: struct { Register, Register, u6 }, + JK: struct { Register, Register }, + JUd5Sk12: struct { Register, u5, i12 }, + Ud15: struct { u15 }, + DJKUa2: struct { Register, Register, Register, u2 }, + DJK: struct { Register, Register, Register }, + DJ: struct { Register, Register }, + EMPTY, + DJUk5Um5: struct { Register, Register, u5, u5 }, + JUd5: struct { Register, u5 }, + JSd5Sk16: struct { Register, i5, i16 }, + DSj20: struct { Register, i20 }, + DJUk14: struct { Register, Register, u14 }, + DJSk16: struct { Register, Register, i16 }, + DJKUa3: struct { Register, Register, Register, u3 }, + DJUk5: struct { Register, Register, u5 }, + DJSk14: struct { Register, Register, i14 }, + JKUd5: struct { Register, Register, u5 }, + DJUk12: struct { Register, Register, u12 }, + DUj5: struct { Register, u5 }, + DJUk6Um6: struct { Register, Register, u6, u6 }, + DJSk12: struct { Register, Register, i12 }, + Sd10Sk16: struct { i10, i16 }, + JUk8: struct { Register, u8 }, + DJKA: struct { Register, Register, Register, Register }, + + pub fn enc(self: Data) u32 { + return switch (self) { + .DJUk8 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u8, @bitCast(data[2]))) << 10), + .DJUk6 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u6, @bitCast(data[2]))) << 10), + .JK => |data| (@as(u32, data[0].enc()) << 5) | (@as(u32, data[1].enc()) << 10), + .JUd5Sk12 => |data| (@as(u32, data[0].enc()) << 5) | @as(u32, @intCast(@as(u5, @bitCast(data[1])))) | (@as(u32, @as(u12, @bitCast(data[2]))) << 10), + .Ud15 => |data| @as(u32, @intCast(@as(u15, @bitCast(data[0])))), + .DJKUa2 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, data[2].enc()) << 10) | (@as(u32, @as(u2, @bitCast(data[3]))) << 15), + .DJK => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, data[2].enc()) << 10), + .DJ => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5), + .EMPTY => 0, + .DJUk5Um5 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u5, @bitCast(data[2]))) << 10) | (@as(u32, @as(u5, @bitCast(data[3]))) << 16), + .JUd5 => |data| (@as(u32, data[0].enc()) << 5) | @as(u32, @intCast(@as(u5, @bitCast(data[1])))), + .JSd5Sk16 => |data| (@as(u32, data[0].enc()) << 5) | @as(u32, @intCast(@as(u5, @bitCast(data[1])))) | (@as(u32, @as(u16, @bitCast(data[2]))) << 10), + .DSj20 => |data| data[0].enc() | (@as(u32, @as(u20, @bitCast(data[1]))) << 5), + .DJUk14 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u14, @bitCast(data[2]))) << 10), + .DJSk16 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u16, @bitCast(data[2]))) << 10), + .DJKUa3 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, data[2].enc()) << 10) | (@as(u32, @as(u3, @bitCast(data[3]))) << 15), + .DJUk5 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u5, @bitCast(data[2]))) << 10), + .DJSk14 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u14, @bitCast(data[2]))) << 10), + .JKUd5 => |data| (@as(u32, data[0].enc()) << 5) | (@as(u32, data[1].enc()) << 10) | @as(u32, @intCast(@as(u5, @bitCast(data[2])))), + .DJUk12 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u12, @bitCast(data[2]))) << 10), + .DUj5 => |data| data[0].enc() | (@as(u32, @as(u5, @bitCast(data[1]))) << 5), + .DJUk6Um6 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u6, @bitCast(data[2]))) << 10) | (@as(u32, @as(u6, @bitCast(data[3]))) << 16), + .DJSk12 => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, @as(u12, @bitCast(data[2]))) << 10), + .Sd10Sk16 => |data| @as(u32, @intCast(@as(u10, @bitCast(data[0])))) | (@as(u32, @as(u16, @bitCast(data[1]))) << 10), + .JUk8 => |data| (@as(u32, data[0].enc()) << 5) | (@as(u32, @as(u8, @bitCast(data[1]))) << 10), + .DJKA => |data| data[0].enc() | (@as(u32, data[1].enc()) << 5) | (@as(u32, data[2].enc()) << 10) | (@as(u32, data[3].enc()) << 15), + }; + } +}; + +pub const Format = @import("std").meta.Tag(Data); + +pub const Inst = struct { + opcode: OpCode, + data: Data, + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn add_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .add_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn add_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .add_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn addi_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .addi_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn addi_w(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .addi_w, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn addu16i_d(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .addu16i_d, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_db_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_db_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_db_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_db_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amadd_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amadd_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amand_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amand_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amand_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amand_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amand_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amand_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amand_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amand_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_db_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_db_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_db_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_db_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amcas_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amcas_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_db_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_db_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammax_db_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammax_db_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_db_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_db_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ammin_db_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ammin_db_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amor_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amor_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amor_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amor_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amor_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amor_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amor_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amor_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_db_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_db_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_db_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_db_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amswap_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amswap_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amxor_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amxor_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amxor_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amxor_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amxor_db_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amxor_db_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn amxor_db_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .amxor_db_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn @"and"(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .@"and", .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn andi(f0: Register, f1: Register, f2: u12) Inst { + return .{ .opcode = .andi, .data = .{ .DJUk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn andn(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .andn, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn asrtgt(f0: Register, f1: Register) Inst { + return .{ .opcode = .asrtgt, .data = .{ .JK = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn asrtle(f0: Register, f1: Register) Inst { + return .{ .opcode = .asrtle, .data = .{ .JK = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn b(f0: i10, f1: i16) Inst { + return .{ .opcode = .b, .data = .{ .Sd10Sk16 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bceqz(f0: Register, f1: i5, f2: i16) Inst { + return .{ .opcode = .bceqz, .data = .{ .JSd5Sk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bcnez(f0: Register, f1: i5, f2: i16) Inst { + return .{ .opcode = .bcnez, .data = .{ .JSd5Sk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn beq(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .beq, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn beqz(f0: Register, f1: i5, f2: i16) Inst { + return .{ .opcode = .beqz, .data = .{ .JSd5Sk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bgt(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .bgt, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bgtu(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .bgtu, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bl(f0: i10, f1: i16) Inst { + return .{ .opcode = .bl, .data = .{ .Sd10Sk16 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ble(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .ble, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bleu(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .bleu, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bne(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .bne, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bnez(f0: Register, f1: i5, f2: i16) Inst { + return .{ .opcode = .bnez, .data = .{ .JSd5Sk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn @"break"(f0: u15) Inst { + return .{ .opcode = .@"break", .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bstrins_d(f0: Register, f1: Register, f2: u6, f3: u6) Inst { + return .{ .opcode = .bstrins_d, .data = .{ .DJUk6Um6 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bstrins_w(f0: Register, f1: Register, f2: u5, f3: u5) Inst { + return .{ .opcode = .bstrins_w, .data = .{ .DJUk5Um5 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bstrpick_d(f0: Register, f1: Register, f2: u6, f3: u6) Inst { + return .{ .opcode = .bstrpick_d, .data = .{ .DJUk6Um6 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn bstrpick_w(f0: Register, f1: Register, f2: u5, f3: u5) Inst { + return .{ .opcode = .bstrpick_w, .data = .{ .DJUk5Um5 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cacop(f0: Register, f1: u5, f2: i12) Inst { + return .{ .opcode = .cacop, .data = .{ .JUd5Sk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn catpick_d(f0: Register, f1: Register, f2: Register, f3: u3) Inst { + return .{ .opcode = .catpick_d, .data = .{ .DJKUa3 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn catpick_w(f0: Register, f1: Register, f2: Register, f3: u2) Inst { + return .{ .opcode = .catpick_w, .data = .{ .DJKUa2 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn clo_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .clo_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn clo_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .clo_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn clz_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .clz_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn clz_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .clz_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cpucfg(f0: Register, f1: Register) Inst { + return .{ .opcode = .cpucfg, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crc_w_b_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crc_w_b_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crc_w_d_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crc_w_d_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crc_w_h_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crc_w_h_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crc_w_w_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crc_w_w_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crcc_w_b_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crcc_w_b_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crcc_w_d_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crcc_w_d_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crcc_w_h_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crcc_w_h_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn crcc_w_w_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .crcc_w_w_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn csrxchg(f0: Register, f1: Register, f2: u14) Inst { + return .{ .opcode = .csrxchg, .data = .{ .DJUk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cto_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .cto_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cto_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .cto_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ctz_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ctz_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ctz_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .ctz_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cu32i_d(f0: Register, f1: i20) Inst { + return .{ .opcode = .cu32i_d, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn cu52i_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .cu52i_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn dbar(f0: u15) Inst { + return .{ .opcode = .dbar, .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn dbgcall(f0: u15) Inst { + return .{ .opcode = .dbgcall, .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn div_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .div_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn div_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .div_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn div_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .div_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn div_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .div_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn eret() Inst { + return .{ .opcode = .eret, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fabs_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fabs_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fabs_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fabs_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fadd_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fadd_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fadd_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fadd_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fclass_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fclass_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fclass_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fclass_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_caf_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_caf_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_caf_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_caf_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_ceq_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_ceq_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_ceq_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_ceq_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cle_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cle_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_clt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_clt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_clt_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_clt_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cne_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cne_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cne_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cne_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cor_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cor_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cor_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cor_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cueq_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cueq_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cueq_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cueq_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cule_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cule_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cule_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cule_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cult_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cult_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cult_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cult_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cun_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cun_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cun_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cun_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cune_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cune_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_cune_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_cune_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_saf_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_saf_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_saf_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_saf_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_seq_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_seq_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_seq_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_seq_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sle_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sle_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_slt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_slt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_slt_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_slt_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sne_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sne_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sne_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sne_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sor_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sor_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sor_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sor_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sueq_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sueq_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sueq_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sueq_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sule_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sule_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sule_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sule_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sult_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sult_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sult_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sult_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sun_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sun_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sun_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sun_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sune_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sune_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcmp_sune_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcmp_sune_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcopysign_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcopysign_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcopysign_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fcopysign_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcsrrd(f0: Register, f1: u5) Inst { + return .{ .opcode = .fcsrrd, .data = .{ .DUj5 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcsrwr(f0: Register, f1: u5) Inst { + return .{ .opcode = .fcsrwr, .data = .{ .JUd5 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcvt_d_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fcvt_d_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fcvt_s_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fcvt_s_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fdiv_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fdiv_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fdiv_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fdiv_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ffint_d_l(f0: Register, f1: Register) Inst { + return .{ .opcode = .ffint_d_l, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ffint_d_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .ffint_d_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ffint_s_l(f0: Register, f1: Register) Inst { + return .{ .opcode = .ffint_s_l, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ffint_s_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .ffint_s_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fld_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .fld_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fld_s(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .fld_s, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldgt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldgt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldgt_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldgt_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldle_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldle_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldx_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldx_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fldx_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fldx_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn flogb_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .flogb_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn flogb_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .flogb_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmadd_d(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fmadd_d, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmadd_s(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fmadd_s, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmax_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmax_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmax_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmax_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmaxa_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmaxa_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmaxa_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmaxa_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmin_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmin_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmin_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmin_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmina_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmina_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmina_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmina_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmov_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fmov_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmov_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fmov_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmsub_d(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fmsub_d, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmsub_s(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fmsub_s, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmul_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmul_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fmul_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fmul_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fneg_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fneg_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fneg_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fneg_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fnmadd_d(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fnmadd_d, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fnmadd_s(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fnmadd_s, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fnmsub_d(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fnmsub_d, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fnmsub_s(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fnmsub_s, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frecip_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .frecip_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frecip_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .frecip_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frecipe_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .frecipe_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frecipe_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .frecipe_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frint_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .frint_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frint_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .frint_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frsqrt_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .frsqrt_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frsqrt_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .frsqrt_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frsqrte_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .frsqrte_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn frsqrte_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .frsqrte_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fscaleb_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fscaleb_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fscaleb_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fscaleb_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fsel(f0: Register, f1: Register, f2: Register, f3: Register) Inst { + return .{ .opcode = .fsel, .data = .{ .DJKA = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fsqrt_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .fsqrt_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fsqrt_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .fsqrt_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fst_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .fst_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fst_s(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .fst_s, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstgt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstgt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstgt_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstgt_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstle_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstle_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstx_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstx_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fstx_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fstx_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fsub_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fsub_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn fsub_s(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .fsub_s, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftint_l_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftint_l_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftint_l_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftint_l_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftint_w_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftint_w_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftint_w_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftint_w_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrm_l_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrm_l_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrm_l_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrm_l_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrm_w_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrm_w_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrm_w_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrm_w_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrne_l_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrne_l_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrne_l_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrne_l_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrne_w_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrne_w_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrne_w_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrne_w_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrp_l_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrp_l_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrp_l_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrp_l_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrp_w_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrp_w_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrp_w_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrp_w_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrz_l_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrz_l_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrz_l_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrz_l_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrz_w_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrz_w_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ftintrz_w_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .ftintrz_w_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ibar(f0: u15) Inst { + return .{ .opcode = .ibar, .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn idle(f0: u15) Inst { + return .{ .opcode = .idle, .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrrd_b(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrrd_b, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrrd_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrrd_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrrd_h(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrrd_h, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrrd_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrrd_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrwr_b(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrwr_b, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrwr_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrwr_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrwr_h(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrwr_h, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn iocsrwr_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .iocsrwr_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn jirl(f0: Register, f1: Register, f2: i16) Inst { + return .{ .opcode = .jirl, .data = .{ .DJSk16 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_b(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_b, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_bu(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_bu, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_h(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_h, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_hu(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_hu, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_w(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_w, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ld_wu(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .ld_wu, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn lddir(f0: Register, f1: Register, f2: u8) Inst { + return .{ .opcode = .lddir, .data = .{ .DJUk8 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldgt_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldgt_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldgt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldgt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldgt_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldgt_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldgt_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldgt_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldle_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldle_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldle_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldle_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldle_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldle_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldox4_d(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .ldox4_d, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldox4_w(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .ldox4_w, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldpte(f0: Register, f1: u8) Inst { + return .{ .opcode = .ldpte, .data = .{ .JUk8 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_bu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_bu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_hu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_hu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ldx_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .ldx_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ll_d(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .ll_d, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ll_w(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .ll_w, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn llacq_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .llacq_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn llacq_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .llacq_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn lu12i_w(f0: Register, f1: i20) Inst { + return .{ .opcode = .lu12i_w, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn maskeqz(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .maskeqz, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn masknez(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .masknez, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mod_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mod_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mod_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mod_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mod_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mod_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mod_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mod_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfcc2fr(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfcc2fr, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfcc2gr(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfcc2gr, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfr2fcc(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfr2fcc, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfr2gr_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfr2gr_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfr2gr_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfr2gr_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movfrh2gr_s(f0: Register, f1: Register) Inst { + return .{ .opcode = .movfrh2gr_s, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movgr2fcc(f0: Register, f1: Register) Inst { + return .{ .opcode = .movgr2fcc, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movgr2fr_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .movgr2fr_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movgr2fr_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .movgr2fr_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn movgr2frh_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .movgr2frh_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mul_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mul_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mul_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mul_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulh_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulh_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulh_du(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulh_du, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulh_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulh_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulh_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulh_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulw_d_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulw_d_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn mulw_d_wu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .mulw_d_wu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn nor(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .nor, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn @"or"(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .@"or", .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn ori(f0: Register, f1: Register, f2: u12) Inst { + return .{ .opcode = .ori, .data = .{ .DJUk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn orn(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .orn, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn pcaddu12i(f0: Register, f1: i20) Inst { + return .{ .opcode = .pcaddu12i, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn pcaddu18i(f0: Register, f1: i20) Inst { + return .{ .opcode = .pcaddu18i, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn pcaddu2i(f0: Register, f1: i20) Inst { + return .{ .opcode = .pcaddu2i, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn pcalau12i(f0: Register, f1: i20) Inst { + return .{ .opcode = .pcalau12i, .data = .{ .DSj20 = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn preld(f0: Register, f1: u5, f2: i12) Inst { + return .{ .opcode = .preld, .data = .{ .JUd5Sk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn preldx(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .preldx, .data = .{ .JKUd5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rdtime_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .rdtime_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rdtimeh_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .rdtimeh_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rdtimel_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .rdtimel_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revb_2h(f0: Register, f1: Register) Inst { + return .{ .opcode = .revb_2h, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revb_2w(f0: Register, f1: Register) Inst { + return .{ .opcode = .revb_2w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revb_4h(f0: Register, f1: Register) Inst { + return .{ .opcode = .revb_4h, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revb_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .revb_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revbit_4b(f0: Register, f1: Register) Inst { + return .{ .opcode = .revbit_4b, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revbit_8b(f0: Register, f1: Register) Inst { + return .{ .opcode = .revbit_8b, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revbit_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .revbit_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revbit_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .revbit_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revh_2w(f0: Register, f1: Register) Inst { + return .{ .opcode = .revh_2w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn revh_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .revh_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rotr_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .rotr_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rotr_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .rotr_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rotri_d(f0: Register, f1: Register, f2: u6) Inst { + return .{ .opcode = .rotri_d, .data = .{ .DJUk6 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn rotri_w(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .rotri_w, .data = .{ .DJUk5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sc_d(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .sc_d, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sc_q(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sc_q, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sc_w(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .sc_w, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn screl_d(f0: Register, f1: Register) Inst { + return .{ .opcode = .screl_d, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn screl_w(f0: Register, f1: Register) Inst { + return .{ .opcode = .screl_w, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sext_b(f0: Register, f1: Register) Inst { + return .{ .opcode = .sext_b, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sext_h(f0: Register, f1: Register) Inst { + return .{ .opcode = .sext_h, .data = .{ .DJ = .{ f0, f1 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sladd_d(f0: Register, f1: Register, f2: Register, f3: u2) Inst { + return .{ .opcode = .sladd_d, .data = .{ .DJKUa2 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sladd_w(f0: Register, f1: Register, f2: Register, f3: u2) Inst { + return .{ .opcode = .sladd_w, .data = .{ .DJKUa2 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sladd_wu(f0: Register, f1: Register, f2: Register, f3: u2) Inst { + return .{ .opcode = .sladd_wu, .data = .{ .DJKUa2 = .{ f0, f1, f2, f3 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sll_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sll_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sll_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sll_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn slli_d(f0: Register, f1: Register, f2: u6) Inst { + return .{ .opcode = .slli_d, .data = .{ .DJUk6 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn slli_w(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .slli_w, .data = .{ .DJUk5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn slt(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .slt, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn slti(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .slti, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sltu(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sltu, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sltui(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .sltui, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sra_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sra_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sra_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sra_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srai_d(f0: Register, f1: Register, f2: u6) Inst { + return .{ .opcode = .srai_d, .data = .{ .DJUk6 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srai_w(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .srai_w, .data = .{ .DJUk5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srl_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .srl_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srl_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .srl_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srli_d(f0: Register, f1: Register, f2: u6) Inst { + return .{ .opcode = .srli_d, .data = .{ .DJUk6 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn srli_w(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .srli_w, .data = .{ .DJUk5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn st_b(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .st_b, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn st_d(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .st_d, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn st_h(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .st_h, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn st_w(f0: Register, f1: Register, f2: i12) Inst { + return .{ .opcode = .st_w, .data = .{ .DJSk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stgt_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stgt_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stgt_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stgt_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stgt_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stgt_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stgt_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stgt_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stle_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stle_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stle_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stle_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stle_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stle_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stle_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stle_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stox4_d(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .stox4_d, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stox4_w(f0: Register, f1: Register, f2: i14) Inst { + return .{ .opcode = .stox4_w, .data = .{ .DJSk14 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stx_b(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stx_b, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stx_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stx_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stx_h(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stx_h, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn stx_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .stx_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sub_d(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sub_d, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn sub_w(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .sub_w, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn syscall(f0: u15) Inst { + return .{ .opcode = .syscall, .data = .{ .Ud15 = .{f0} } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbclr() Inst { + return .{ .opcode = .tlbclr, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbfill() Inst { + return .{ .opcode = .tlbfill, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbflush() Inst { + return .{ .opcode = .tlbflush, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbinv(f0: Register, f1: Register, f2: u5) Inst { + return .{ .opcode = .tlbinv, .data = .{ .JKUd5 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbrd() Inst { + return .{ .opcode = .tlbrd, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbsrch() Inst { + return .{ .opcode = .tlbsrch, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn tlbwr() Inst { + return .{ .opcode = .tlbwr, .data = .EMPTY }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn xor(f0: Register, f1: Register, f2: Register) Inst { + return .{ .opcode = .xor, .data = .{ .DJK = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn xori(f0: Register, f1: Register, f2: u12) Inst { + return .{ .opcode = .xori, .data = .{ .DJUk12 = .{ f0, f1, f2 } } }; + } + + // Workaround https://github.com/ziglang/zig/issues/24127 + pub noinline fn xxx_unknown_1() Inst { + return .{ .opcode = .xxx_unknown_1, .data = .EMPTY }; + } +}; diff --git a/src/arch/loongarch/utils.zig b/src/arch/loongarch/utils.zig new file mode 100644 index 000000000000..17cff538d4cd --- /dev/null +++ b/src/arch/loongarch/utils.zig @@ -0,0 +1,40 @@ +const std = @import("std"); +const mem = std.mem; + +pub fn page(val: u64) u64 { + return val & ~0xfff; +} + +pub fn hi20(val: u64, pc: u64) i20 { + return @bitCast(@as(i20, @truncate((@as(i64, @intCast(val)) - @as(i64, @intCast(pc)) + 0x800) >> 12))); +} + +pub fn lo12(val: u64, pc: u64) i12 { + return @bitCast(@as(i12, @truncate((@as(i64, @intCast(val)) - @as(i64, @intCast(pc))) & 0xfff))); +} + +pub fn notZero(val: anytype) ?@TypeOf(val) { + return if (val != 0) val else null; +} + +pub const ImmOffset = enum(u5) { + d = 0, + j = 5, + k = 10, + a = 15, + m = 16, + n = 18, + + pub inline fn offset(off: ImmOffset) u5 { + return @intFromEnum(off); + } +}; + +pub fn relocImm(inst: u32, comptime slot: ImmOffset, T: type, val: T) u32 { + const uT = std.meta.Int(.unsigned, @bitSizeOf(T)); + const off = slot.offset(); + const slot_size: u5 = @intCast(@typeInfo(T).int.bits); + const mask: u32 = (~((@as(u32, 1) << (off + slot_size)) - 1)) | ((1 << off) - 1); + const val_u32: u32 = @as(uT, @bitCast(val)); + return (inst & mask) | (val_u32 << off); +} diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index b9a16d5a75f8..c07fddcbb52b 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -1015,7 +1015,7 @@ fn formatAir( _: std.fmt.FormatOptions, writer: anytype, ) @TypeOf(writer).Error!void { - data.func.air.dumpInst(data.inst, data.func.pt, data.func.liveness); + data.func.air.writeInst(writer, data.inst, data.func.pt, data.func.liveness); } fn fmtAir(func: *Func, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) { return .{ .data = .{ .func = func, .inst = inst } }; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 6a6022b62d27..266036ea1398 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1121,7 +1121,7 @@ fn formatAir( _: std.fmt.FormatOptions, writer: anytype, ) @TypeOf(writer).Error!void { - data.self.air.dumpInst(data.inst, data.self.pt, data.self.liveness); + data.self.air.writeInst(writer, data.inst, data.self.pt, data.self.liveness); } fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) { return .{ .data = .{ .self = self, .inst = inst } }; diff --git a/src/codegen.zig b/src/codegen.zig index 11b6eedc8683..0c88a7afb47c 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -41,6 +41,7 @@ fn devFeatureForBackend(backend: std.builtin.CompilerBackend) dev.Feature { .stage2_wasm => .wasm_backend, .stage2_x86 => .x86_backend, .stage2_x86_64 => .x86_64_backend, + .stage2_loongarch => .loongarch_backend, _ => unreachable, }; } @@ -52,6 +53,7 @@ fn importBackend(comptime backend: std.builtin.CompilerBackend) type { .stage2_arm => @import("arch/arm/CodeGen.zig"), .stage2_c => @import("codegen/c.zig"), .stage2_llvm => @import("codegen/llvm.zig"), + .stage2_loongarch => @import("arch/loongarch/CodeGen.zig"), .stage2_powerpc => @import("arch/powerpc/CodeGen.zig"), .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"), .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"), @@ -77,6 +79,7 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co .stage2_riscv64, .stage2_sparc64, .stage2_spirv, + .stage2_loongarch, .stage2_powerpc, => |backend| { dev.check(devFeatureForBackend(backend)); @@ -91,6 +94,7 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co pub const AnyMir = union { aarch64: @import("arch/aarch64/Mir.zig"), arm: @import("arch/arm/Mir.zig"), + loongarch: @import("arch/loongarch/Mir.zig"), powerpc: noreturn, //@import("arch/powerpc/Mir.zig"), riscv64: @import("arch/riscv64/Mir.zig"), sparc64: @import("arch/sparc64/Mir.zig"), @@ -102,6 +106,7 @@ pub const AnyMir = union { return switch (backend) { .stage2_aarch64 => "aarch64", .stage2_arm => "arm", + .stage2_loongarch => "loongarch", .stage2_powerpc => "powerpc", .stage2_riscv64 => "riscv64", .stage2_sparc64 => "sparc64", @@ -119,6 +124,7 @@ pub const AnyMir = union { else => unreachable, inline .stage2_aarch64, .stage2_arm, + .stage2_loongarch, .stage2_powerpc, .stage2_riscv64, .stage2_sparc64, @@ -150,6 +156,7 @@ pub fn generateFunction( else => unreachable, inline .stage2_aarch64, .stage2_arm, + .stage2_loongarch, .stage2_powerpc, .stage2_riscv64, .stage2_sparc64, @@ -188,6 +195,7 @@ pub fn emitFunction( else => unreachable, inline .stage2_aarch64, .stage2_arm, + .stage2_loongarch, .stage2_powerpc, .stage2_riscv64, .stage2_sparc64, diff --git a/src/dev.zig b/src/dev.zig index 25c2d01a4ba4..096f1e943b8e 100644 --- a/src/dev.zig +++ b/src/dev.zig @@ -48,6 +48,10 @@ pub const Env = enum { /// - `zig build-* -fno-llvm -fno-lld -target wasm32-* --listen=-` wasm, + /// - sema + /// - `zig build-* -fincremental -fno-llvm -fno-lld -target loongarch{32,64}-linux --listen=-` + @"loongarch-linux", + pub inline fn supports(comptime dev_env: Env, comptime feature: Feature) bool { return switch (dev_env) { .full => true, @@ -89,6 +93,7 @@ pub const Env = enum { .riscv64_backend, .sparc64_backend, .spirv64_backend, + .loongarch_backend, .lld_linker, .coff_linker, .elf_linker, @@ -196,6 +201,16 @@ pub const Env = enum { => true, else => Env.sema.supports(feature), }, + .@"loongarch-linux" => switch (feature) { + .build_command, + .stdio_listen, + .incremental, + .loongarch_backend, + .elf_linker, + .legalize, + => true, + else => Env.sema.supports(feature), + }, }; } @@ -259,6 +274,7 @@ pub const Feature = enum { riscv64_backend, sparc64_backend, spirv64_backend, + loongarch_backend, lld_linker, coff_linker, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index dc27e0bdd760..51162a3ffd20 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -4128,7 +4128,7 @@ pub fn getTarget(self: *const Elf) *const std.Target { fn requiresThunks(self: Elf) bool { return switch (self.getTarget().cpu.arch) { .aarch64 => true, - .x86_64, .riscv64 => false, + .x86_64, .riscv64, .loongarch64, .loongarch32 => false, else => @panic("TODO unimplemented architecture"), }; } @@ -4441,7 +4441,7 @@ fn createThunks(elf_file: *Elf, atom_list: *AtomList) !void { _ = math.cast(i28, taddr + rel.r_addend - saddr) orelse break :r false; break :r true; }, - .x86_64, .riscv64 => unreachable, + .x86_64, .riscv64, .loongarch64, .loongarch32 => unreachable, else => @panic("unsupported arch"), }; if (is_reachable) continue; diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 0869d6582e2f..93ffd97251df 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -354,6 +354,10 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype error.RelocFailure => has_reloc_errors = true, else => |e| return e, }, + .loongarch64, .loongarch32 => loongarch.scanReloc(self, elf_file, rel, symbol, code, &it) catch |err| switch (err) { + error.RelocFailure => has_reloc_errors = true, + else => |e| return e, + }, else => return error.UnsupportedCpuArch, } } @@ -535,8 +539,9 @@ fn reportTextRelocError( ) RelocError!void { const diags = &elf_file.base.comp.link_diags; var err = try diags.addErrorWithNotes(1); - try err.addMsg("relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + try err.addMsg("relocation at offset 0x{x} with type {} against symbol '{s}' cannot be used", .{ rel.r_offset, + relocation.fmtRelocType(rel.r_type(), elf_file.getTarget().cpu.arch), symbol.name(elf_file), }); err.addNote("in {}:{s}", .{ self.file(elf_file).?.fmtPath(), self.name(elf_file) }); @@ -551,8 +556,9 @@ fn reportPicError( ) RelocError!void { const diags = &elf_file.base.comp.link_diags; var err = try diags.addErrorWithNotes(2); - try err.addMsg("relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + try err.addMsg("relocation at offset 0x{x} with type {} against symbol '{s}' cannot be used", .{ rel.r_offset, + relocation.fmtRelocType(rel.r_type(), elf_file.getTarget().cpu.arch), symbol.name(elf_file), }); err.addNote("in {}:{s}", .{ self.file(elf_file).?.fmtPath(), self.name(elf_file) }); @@ -568,8 +574,9 @@ fn reportNoPicError( ) RelocError!void { const diags = &elf_file.base.comp.link_diags; var err = try diags.addErrorWithNotes(2); - try err.addMsg("relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + try err.addMsg("relocation at offset 0x{x} with type {} against symbol '{s}' cannot be used", .{ rel.r_offset, + relocation.fmtRelocType(rel.r_type(), elf_file.getTarget().cpu.arch), symbol.name(elf_file), }); err.addNote("in {}:{s}", .{ self.file(elf_file).?.fmtPath(), self.name(elf_file) }); @@ -688,6 +695,12 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi => has_reloc_errors = true, else => |e| return e, }, + .loongarch64, .loongarch32 => loongarch.resolveRelocAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { + error.RelocFailure, + error.RelaxFailure, + => has_reloc_errors = true, + else => |e| return e, + }, else => return error.UnsupportedCpuArch, } } @@ -878,6 +891,10 @@ pub fn resolveRelocsNonAlloc(self: Atom, elf_file: *Elf, code: []u8, undefs: any error.RelocFailure => has_reloc_errors = true, else => |e| return e, }, + .loongarch64, .loongarch32 => loongarch.resolveRelocNonAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { + error.RelocFailure => has_reloc_errors = true, + else => |e| return e, + }, else => return error.UnsupportedCpuArch, } } @@ -1885,6 +1902,8 @@ const riscv = struct { .SUB_ULEB128, .SET_ULEB128, + + .RELAX, => {}, else => try atom.reportUnhandledRelocError(rel, elf_file), @@ -1912,6 +1931,7 @@ const riscv = struct { switch (r_type) { .NONE => unreachable, + .RELAX => {}, .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), @@ -2051,6 +2071,7 @@ const riscv = struct { switch (r_type) { .NONE => unreachable, + .RELAX => {}, .@"32" => try cwriter.writeInt(i32, @as(i32, @intCast(S + A)), .little), .@"64" => if (atom.debugTombstoneValue(target.*, elf_file)) |value| @@ -2084,6 +2105,201 @@ const riscv = struct { const riscv_util = @import("../riscv.zig"); }; +const loongarch = struct { + fn scanReloc( + atom: Atom, + elf_file: *Elf, + rel: elf.Elf64_Rela, + symbol: *Symbol, + code: ?[]const u8, + it: *RelocsIterator, + ) !void { + _ = code; + _ = it; + + const r_type: elf.R_LARCH = @enumFromInt(rel.r_type()); + const is_dyn_lib = elf_file.isEffectivelyDynLib(); + + switch (r_type) { + .@"32" => try atom.scanReloc(symbol, rel, absRelocAction(symbol, elf_file), elf_file), + .@"64" => try atom.scanReloc(symbol, rel, dynAbsRelocAction(symbol, elf_file), elf_file), + .@"32_PCREL" => try atom.scanReloc(symbol, rel, pcRelocAction(symbol, elf_file), elf_file), + .@"64_PCREL" => try atom.scanReloc(symbol, rel, pcRelocAction(symbol, elf_file), elf_file), + + .B26, .PCALA_HI20, .CALL36 => if (symbol.flags.import) { + symbol.flags.needs_plt = true; + }, + .GOT_HI20, .GOT_PC_HI20 => symbol.flags.needs_got = true, + .TLS_IE_HI20, .TLS_IE_PC_HI20 => symbol.flags.needs_gottp = true, + .TLS_GD_PC_HI20, .TLS_LD_PC_HI20, .TLS_GD_HI20, .TLS_LD_HI20 => symbol.flags.needs_tlsgd = true, + .TLS_DESC_CALL => symbol.flags.needs_tlsdesc = true, + + .TLS_LE_HI20, + .TLS_LE_LO12, + .TLS_LE64_LO20, + .TLS_LE64_HI12, + .TLS_LE_HI20_R, + .TLS_LE_LO12_R, + => if (is_dyn_lib) try atom.reportPicError(symbol, rel, elf_file), + + .B16, + .B21, + .ABS_HI20, + .ABS_LO12, + .ABS64_LO20, + .ABS64_HI12, + .PCALA_LO12, + .PCALA64_LO20, + .PCALA64_HI12, + .GOT_PC_LO12, + .GOT64_PC_LO20, + .GOT64_PC_HI12, + .GOT_LO12, + .GOT64_LO20, + .GOT64_HI12, + .TLS_IE_PC_LO12, + .TLS_IE64_PC_LO20, + .TLS_IE64_PC_HI12, + .TLS_IE_LO12, + .TLS_IE64_LO20, + .TLS_IE64_HI12, + .ADD6, + .SUB6, + .ADD8, + .SUB8, + .ADD16, + .SUB16, + .ADD32, + .SUB32, + .ADD64, + .SUB64, + .ADD_ULEB128, + .SUB_ULEB128, + .TLS_DESC_PC_HI20, + .TLS_DESC_PC_LO12, + .TLS_DESC_LD, + .TLS_LE_ADD_R, + .RELAX, + => {}, + + else => try atom.reportUnhandledRelocError(rel, elf_file), + } + } + + fn resolveRelocAlloc( + atom: Atom, + elf_file: *Elf, + rel: elf.Elf64_Rela, + target: *const Symbol, + args: ResolveArgs, + it: *RelocsIterator, + code: []u8, + stream: anytype, + ) !void { + const diags = &elf_file.base.comp.link_diags; + const r_type: elf.R_LARCH = @enumFromInt(rel.r_type()); + const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; + const cwriter = stream.writer(); + const code_buf = code[r_offset..]; + + const P, const A, const S, const GOT, const G, const TP, const DTP = args; + _ = TP; + _ = DTP; + _ = GOT; + _ = G; + _ = diags; + _ = it; + + switch (r_type) { + .NONE => unreachable, + .RELAX => {}, + + .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), + + .@"64" => { + try atom.resolveDynAbsReloc( + target, + rel, + dynAbsRelocAction(target, elf_file), + elf_file, + cwriter, + ); + }, + + .CALL36 => { + // TODO: relax + const val = S + A - P; + try writeImmSlot(code_buf[0..4], .j, i20, .cast, (val + 0x20000) >> 18); + try writeImmSlot(code_buf[4..][0..4], .k, i16, .trunc, val >> 2); + }, + + .PCALA_HI20 => { + // TODO: relax + try writeImmSlot(code_buf[0..4], .j, i20, .cast, (S + A + 0x800 - P) >> 12); + }, + .PCALA_LO12 => { + // TODO: relax + try writeImmSlot(code_buf[0..4], .k, i12, .trunc, S + A); + }, + + else => try atom.reportUnhandledRelocError(rel, elf_file), + } + } + + fn resolveRelocNonAlloc( + atom: Atom, + elf_file: *Elf, + rel: elf.Elf64_Rela, + target: *const Symbol, + args: ResolveArgs, + it: *RelocsIterator, + code: []u8, + stream: anytype, + ) !void { + _ = it; + + const r_type: elf.R_LARCH = @enumFromInt(rel.r_type()); + const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; + const cwriter = stream.writer(); + + _, const A, const S, const GOT, _, _, const DTP = args; + _ = GOT; + _ = DTP; + _ = r_offset; + _ = code; + + switch (r_type) { + .NONE => unreachable, + .RELAX => {}, + + .@"32" => try cwriter.writeInt(i32, @as(i32, @intCast(S + A)), .little), + .@"64" => if (atom.debugTombstoneValue(target.*, elf_file)) |value| + try cwriter.writeInt(u64, value, .little) + else + try cwriter.writeInt(i64, S + A, .little), + + else => try atom.reportUnhandledRelocError(rel, elf_file), + } + } + + fn writeImmSlot( + code: *[4]u8, + comptime slot: la_utils.ImmOffset, + comptime T: anytype, + comptime convert: enum { cast, trunc }, + value: i64, + ) RelocError!void { + const inst: u32 = mem.readInt(u32, code, .little); + const val: T = switch (convert) { + .cast => math.cast(T, value) orelse return error.Overflow, + .trunc => @truncate(value), + }; + mem.writeInt(u32, code, la_utils.relocImm(inst, slot, T, val), .little); + } + + const la_utils = @import("../../arch/loongarch/utils.zig"); +}; + const ResolveArgs = struct { i64, i64, i64, i64, i64, i64, i64 }; const RelocError = error{ diff --git a/src/link/Elf/eh_frame.zig b/src/link/Elf/eh_frame.zig index bf46fb02621d..b40d6f00304f 100644 --- a/src/link/Elf/eh_frame.zig +++ b/src/link/Elf/eh_frame.zig @@ -328,6 +328,7 @@ fn resolveReloc(rec: anytype, sym: *const Symbol, rel: elf.Elf64_Rela, elf_file: .x86_64 => try x86_64.resolveReloc(rec, elf_file, rel, P, S + A, contents[offset..]), .aarch64 => try aarch64.resolveReloc(rec, elf_file, rel, P, S + A, contents[offset..]), .riscv64 => try riscv.resolveReloc(rec, elf_file, rel, P, S + A, contents[offset..]), + .loongarch64, .loongarch32 => try loongarch.resolveReloc(rec, elf_file, rel, P, S + A, contents[offset..]), else => return error.UnsupportedCpuArch, } } @@ -604,6 +605,19 @@ const riscv = struct { } }; +const loongarch = struct { + fn resolveReloc(rec: anytype, elf_file: *Elf, rel: elf.Elf64_Rela, source: i64, target: i64, data: []u8) !void { + const r_type: elf.R_LARCH = @enumFromInt(rel.r_type()); + _ = source; + _ = target; + _ = data; + switch (r_type) { + .NONE => {}, + else => try reportInvalidReloc(rec, elf_file, rel), + } + } +}; + fn reportInvalidReloc(rec: anytype, elf_file: *Elf, rel: elf.Elf64_Rela) !void { const diags = &elf_file.base.comp.link_diags; var err = try diags.addErrorWithNotes(1); diff --git a/src/link/Elf/relocatable.zig b/src/link/Elf/relocatable.zig index 6541364f9235..e1056ba90899 100644 --- a/src/link/Elf/relocatable.zig +++ b/src/link/Elf/relocatable.zig @@ -365,7 +365,6 @@ fn writeSyntheticSections(elf_file: *Elf) !void { const SortRelocs = struct { pub fn lessThan(ctx: void, lhs: elf.Elf64_Rela, rhs: elf.Elf64_Rela) bool { _ = ctx; - assert(lhs.r_offset != rhs.r_offset); return lhs.r_offset < rhs.r_offset; } }; diff --git a/src/link/Elf/relocation.zig b/src/link/Elf/relocation.zig index 047312cd6878..0cbd24305a10 100644 --- a/src/link/Elf/relocation.zig +++ b/src/link/Elf/relocation.zig @@ -73,11 +73,26 @@ const riscv64_relocs = Table(11, elf.R_RISCV, .{ .{ .tlsdesc, .TLSDESC }, }); +const loongarch64_relocs = Table(11, elf.R_LARCH, .{ + .{ .none, .NONE }, + .{ .abs, .@"64" }, + .{ .copy, .COPY }, + .{ .rel, .RELATIVE }, + .{ .irel, .IRELATIVE }, + .{ .glob_dat, .@"64" }, + .{ .jump_slot, .JUMP_SLOT }, + .{ .dtpmod, .TLS_DTPMOD64 }, + .{ .dtpoff, .TLS_DTPREL64 }, + .{ .tpoff, .TLS_TPREL64 }, + .{ .tlsdesc, .TLS_DESC64 }, +}); + pub fn decode(r_type: u32, cpu_arch: std.Target.Cpu.Arch) ?Kind { return switch (cpu_arch) { .x86_64 => x86_64_relocs.decode(r_type), .aarch64 => aarch64_relocs.decode(r_type), .riscv64 => riscv64_relocs.decode(r_type), + .loongarch64 => loongarch64_relocs.decode(r_type), else => @panic("TODO unhandled cpu arch"), }; } @@ -87,6 +102,7 @@ pub fn encode(comptime kind: Kind, cpu_arch: std.Target.Cpu.Arch) u32 { .x86_64 => x86_64_relocs.encode(kind), .aarch64 => aarch64_relocs.encode(kind), .riscv64 => riscv64_relocs.encode(kind), + .loongarch64 => loongarch64_relocs.encode(kind), else => @panic("TODO unhandled cpu arch"), }; } @@ -102,6 +118,10 @@ pub const dwarf = struct { .@"32" => elf.R_RISCV.@"32", .@"64" => .@"64", }), + .loongarch64, .loongarch32 => @intFromEnum(switch (format) { + .@"32" => elf.R_LARCH.@"32", + .@"64" => .@"64", + }), else => @panic("TODO unhandled cpu arch"), }; } @@ -129,6 +149,14 @@ pub const dwarf = struct { }, .debug_frame => unreachable, })), + .loongarch64, .loongarch32 => @intFromEnum(@as(elf.R_LARCH, switch (source_section) { + else => switch (address_size) { + .@"32" => .@"32", + .@"64" => .@"64", + else => unreachable, + }, + .debug_frame => unreachable, + })), else => @panic("TODO unhandled cpu arch"), }; } @@ -161,6 +189,7 @@ fn formatRelocType( .x86_64 => try writer.print("R_X86_64_{s}", .{@tagName(@as(elf.R_X86_64, @enumFromInt(r_type)))}), .aarch64 => try writer.print("R_AARCH64_{s}", .{@tagName(@as(elf.R_AARCH64, @enumFromInt(r_type)))}), .riscv64 => try writer.print("R_RISCV_{s}", .{@tagName(@as(elf.R_RISCV, @enumFromInt(r_type)))}), + .loongarch64, .loongarch32 => try writer.print("R_LARCH_{s}", .{@tagName(@as(elf.R_LARCH, @enumFromInt(r_type)))}), else => unreachable, } } diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig index aca0c17d0c07..07ddb24e257d 100644 --- a/src/link/Elf/synthetic_sections.zig +++ b/src/link/Elf/synthetic_sections.zig @@ -666,6 +666,7 @@ pub const PltSection = struct { return switch (cpu_arch) { .x86_64 => 32, .aarch64 => 8 * @sizeOf(u32), + .loongarch64 => 8 * @sizeOf(u32), else => @panic("TODO implement preambleSize for this cpu arch"), }; } @@ -674,6 +675,7 @@ pub const PltSection = struct { return switch (cpu_arch) { .x86_64 => 16, .aarch64 => 4 * @sizeOf(u32), + .loongarch64 => 4 * @sizeOf(u32), else => @panic("TODO implement entrySize for this cpu arch"), }; } @@ -683,6 +685,7 @@ pub const PltSection = struct { switch (cpu_arch) { .x86_64 => try x86_64.write(plt, elf_file, writer), .aarch64 => try aarch64.write(plt, elf_file, writer), + .loongarch64 => try loongarch64.write(plt, elf_file, writer), else => return error.UnsupportedCpuArch, } } @@ -873,6 +876,53 @@ pub const PltSection = struct { const Instruction = aarch64_util.Instruction; const Register = aarch64_util.Register; }; + + const loongarch64 = struct { + fn write(plt: PltSection, elf_file: *Elf, writer: anytype) !void { + { + const shdrs = elf_file.sections.items(.shdr); + const plt_addr = shdrs[elf_file.section_indexes.plt.?].sh_addr; + const got_plt_addr = shdrs[elf_file.section_indexes.got_plt.?].sh_addr; + + // .got.plt[2] + // TODO: relax if possible + const preamble = &[_]Lir.Inst{ + .{ .opcode = .pcaddu12i, .data = .{ .DSj20 = .{ .t2, la_utils.hi20(got_plt_addr, plt_addr) } } }, + .{ .opcode = .sub_d, .data = .{ .DJK = .{ .t1, .t1, .r3 } } }, + .{ .opcode = .ld_d, .data = .{ .DJSk12 = .{ .t3, .t2, la_utils.lo12(got_plt_addr, plt_addr) } } }, + .{ .opcode = .addi_d, .data = .{ .DJSk12 = .{ .t1, .t1, -44 } } }, + .{ .opcode = .addi_d, .data = .{ .DJSk12 = .{ .t0, .t2, la_utils.lo12(got_plt_addr, plt_addr) } } }, + .{ .opcode = .srli_d, .data = .{ .DJUk6 = .{ .t1, .t1, 1 } } }, + .{ .opcode = .ld_d, .data = .{ .DJSk12 = .{ .t0, .t0, 8 } } }, + .{ .opcode = .jirl, .data = .{ .DJSk16 = .{ .zero, .t3, 0 } } }, + }; + comptime assert(preamble.len == 8); + for (preamble) |inst| { + try writer.writeInt(u32, inst.encode(), .little); + } + } + + for (plt.symbols.items) |ref| { + const sym = elf_file.symbol(ref).?; + const target_addr: u64 = @intCast(sym.gotPltAddress(elf_file)); + const pc: u64 = @intCast(sym.pltAddress(elf_file)); + + const insts = &[_]Lir.Inst{ + .{ .opcode = .pcaddu12i, .data = .{ .DSj20 = .{ .t3, la_utils.hi20(target_addr, pc) } } }, + .{ .opcode = .ld_d, .data = .{ .DJSk12 = .{ .t3, .t3, la_utils.lo12(target_addr, pc) } } }, + .{ .opcode = .jirl, .data = .{ .DJSk16 = .{ .t1, .t3, 0 } } }, + .{ .opcode = .@"break", .data = .{ .Ud15 = .{0} } }, + }; + comptime assert(insts.len == 4); + for (insts) |inst| { + try writer.writeInt(u32, inst.encode(), .little); + } + } + } + + const la_utils = @import("../../arch/loongarch/utils.zig"); + const Lir = @import("../../arch/loongarch/Lir.zig"); + }; }; pub const GotPltSection = struct { @@ -930,6 +980,7 @@ pub const PltGotSection = struct { return switch (cpu_arch) { .x86_64 => 16, .aarch64 => 4 * @sizeOf(u32), + .loongarch64 => 4 * @sizeOf(u32), else => @panic("TODO implement PltGotSection.entrySize for this arch"), }; } @@ -939,6 +990,7 @@ pub const PltGotSection = struct { switch (cpu_arch) { .x86_64 => try x86_64.write(plt_got, elf_file, writer), .aarch64 => try aarch64.write(plt_got, elf_file, writer), + .loongarch64 => try loongarch64.write(plt_got, elf_file, writer), else => return error.UnsupportedCpuArch, } } @@ -1012,6 +1064,30 @@ pub const PltGotSection = struct { const Instruction = aarch64_util.Instruction; const Register = aarch64_util.Register; }; + + const loongarch64 = struct { + fn write(plt_got: PltGotSection, elf_file: *Elf, writer: anytype) !void { + for (plt_got.symbols.items) |ref| { + const sym = elf_file.symbol(ref).?; + const target_addr: u64 = @intCast(sym.gotAddress(elf_file)); + const pc: u64 = @intCast(sym.pltGotAddress(elf_file)); + + const insts = &[_]Lir.Inst{ + .{ .opcode = .pcaddu12i, .data = .{ .DSj20 = .{ .t3, la_utils.hi20(target_addr, pc) } } }, + .{ .opcode = .ld_d, .data = .{ .DJSk12 = .{ .t3, .t3, la_utils.lo12(target_addr, pc) } } }, + .{ .opcode = .jirl, .data = .{ .DJSk16 = .{ .t1, .t3, 0 } } }, + .{ .opcode = .@"break", .data = .{ .Ud15 = .{0} } }, + }; + comptime assert(insts.len == 4); + for (insts) |inst| { + try writer.writeInt(u32, inst.encode(), .little); + } + } + } + + const la_utils = @import("../../arch/loongarch/utils.zig"); + const Lir = @import("../../arch/loongarch/Lir.zig"); + }; }; pub const CopyRelSection = struct { diff --git a/src/target.zig b/src/target.zig index f96984903780..8dcd9c8e9912 100644 --- a/src/target.zig +++ b/src/target.zig @@ -813,6 +813,7 @@ pub fn zigBackend(target: *const std.Target, use_llvm: bool) std.builtin.Compile return switch (target.cpu.arch) { .aarch64, .aarch64_be => .stage2_aarch64, .arm, .armeb, .thumb, .thumbeb => .stage2_arm, + .loongarch64, .loongarch32 => .stage2_loongarch, .powerpc, .powerpcle, .powerpc64, .powerpc64le => .stage2_powerpc, .riscv64 => .stage2_riscv64, .sparc64 => .stage2_sparc64, @@ -853,7 +854,7 @@ pub inline fn backendSupportsFeature(backend: std.builtin.CompilerBackend, compt }, .separate_thread => switch (backend) { .stage2_llvm => false, - .stage2_c, .stage2_wasm, .stage2_x86_64 => true, + .stage2_c, .stage2_wasm, .stage2_x86_64, .stage2_loongarch => true, // TODO: most self-hosted backends should be able to support this without too much work. else => false, }, diff --git a/tools/gen_loongarch_encoding.zig b/tools/gen_loongarch_encoding.zig new file mode 100644 index 000000000000..81936f90f348 --- /dev/null +++ b/tools/gen_loongarch_encoding.zig @@ -0,0 +1,514 @@ +const std = @import("std"); +const fs = std.fs; +const Allocator = std.mem.Allocator; +const print = std.debug.print; + +pub fn main() anyerror!void { + var arena_allocator = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + var args = try std.process.argsWithAllocator(arena); + const arg0 = args.next().?; + + const opcodes_path = args.next() orelse usageAndExit(arg0, 0); + const zig_path = args.next() orelse usageAndExit(arg0, 1); + + var descs = OpcodeDescList.init(arena); + defer descs.deinit(); + var formats = OpcodeFormatList.init(arena); + defer formats.deinit(); + + var opcodes_dir = try fs.cwd().openDir(opcodes_path, .{ .iterate = true }); + defer opcodes_dir.close(); + var opcodes_iter = opcodes_dir.iterateAssumeFirstIteration(); + while (try opcodes_iter.next()) |opcodes_file| { + if (opcodes_file.kind != .file) continue; + if (!std.mem.endsWith(u8, opcodes_file.name, ".txt")) continue; + + // exclude LSX, LASX, LVZ, LBT for now + if (std.mem.indexOf(u8, opcodes_file.name, "lvz") != null) continue; + if (std.mem.indexOf(u8, opcodes_file.name, "lsx") != null) continue; + if (std.mem.indexOf(u8, opcodes_file.name, "lasx") != null) continue; + if (std.mem.indexOf(u8, opcodes_file.name, "lbt") != null) continue; + + print("Loading desc file: {s}\n", .{opcodes_file.name}); + + const file = try opcodes_dir.openFile(opcodes_file.name, .{}); + defer file.close(); + try loadFromDescFile(&descs, arena, file); + } + std.mem.sort(OpcodeDesc, descs.items, false, struct { + fn cmp(_: bool, lhs: OpcodeDesc, rhs: OpcodeDesc) bool { + return std.mem.order(u8, lhs.name, rhs.name) == .lt; + } + }.cmp); + + print("Loaded {} instructions\n", .{descs.items.len}); + for (descs.items) |desc| { + var format = desc.format; + // wipe out register bank information, because they are the same formats for encoding + inline for (&format) |*slot| { + switch (slot.*) { + else => {}, + .reg => |*reg| { + reg.class = .int; + }, + } + } + try formats.put(format, .{}); + } + print("Found {} formats\n", .{formats.count()}); + + var zig_dir = try fs.cwd().openDir(zig_path, .{}); + defer zig_dir.close(); + const out_file = try zig_dir.createFile("src/arch/loongarch/encoding.zig", .{}); + defer out_file.close(); + const out_writer = out_file.writer(); + + // TODO: include commit hash of loongarch-opcodes + try out_writer.writeAll("// This file is auto-generated by tools/gen_loongarch_encoding.zig\n\n"); + try out_writer.writeAll( + \\const Register = @import("bits.zig").Register; + \\ + \\pub const OpCode = enum { + \\ + ); + for (descs.items) |*desc| { + const name_buf = try arena.dupe(u8, desc.name); + defer arena.free(name_buf); + std.mem.replaceScalar(u8, name_buf, '.', '_'); + + if (std.zig.isValidId(name_buf)) { + try out_writer.print(" {s},\n", .{name_buf}); + } else { + try out_writer.print(" @\"{s}\",\n", .{name_buf}); + } + } + try out_writer.writeAll( + \\ + \\ pub fn enc(opcode: OpCode) u32 { + \\ return switch (opcode) { + \\ + ); + for (descs.items) |*desc| { + const name_buf = try arena.dupe(u8, desc.name); + defer arena.free(name_buf); + std.mem.replaceScalar(u8, name_buf, '.', '_'); + + if (std.zig.isValidId(name_buf)) { + try out_writer.print(" .{s}", .{name_buf}); + } else { + try out_writer.print(" .@\"{s}\"", .{name_buf}); + } + try out_writer.print(" => 0x{x:0>8},\n", .{desc.word}); + } + try out_writer.writeAll( + \\ }; + \\ } + \\}; + \\ + \\pub const Data = union(enum) { + \\ + ); + var format_iter1 = formats.keyIterator(); + while (format_iter1.next()) |format| { + var format_buf = std.ArrayList(u8).init(arena); + defer format_buf.deinit(); + try OpcodeDesc.stringifyFormat(format, format_buf.writer()); + const format_str = format_buf.items; + if (std.mem.eql(u8, format_str, "EMPTY")) { + try out_writer.writeAll( + \\ EMPTY, + \\ + ); + } else { + try out_writer.print(" {s}: struct {{", .{format_str}); + if (format.len != 1) try out_writer.writeAll(" "); + for (format, 0..) |slot, slot_i| { + if (slot == .none) continue; + if (slot_i != 0) try out_writer.writeAll(", "); + switch (slot) { + .none => unreachable, + .reg => |_| { + try out_writer.writeAll("Register"); + }, + .imm => |imm| { + try out_writer.print("{}", .{imm}); + }, + } + } + if (format.len != 1) try out_writer.writeAll(" "); + try out_writer.writeAll("},\n"); + } + } + try out_writer.writeAll( + \\ + \\ pub fn enc(self: Data) u32 { + \\ return switch (self) { + \\ + ); + var format_iter3 = formats.keyIterator(); + while (format_iter3.next()) |format| { + var format_buf = std.ArrayList(u8).init(arena); + defer format_buf.deinit(); + try OpcodeDesc.stringifyFormat(format, format_buf.writer()); + const format_str = format_buf.items; + if (std.mem.eql(u8, format_str, "EMPTY")) { + try out_writer.writeAll( + \\ .EMPTY => 0, + \\ + ); + } else { + try out_writer.print(" .{s} => |data| ", .{format_str}); + for (format, 0..) |slot, slot_i| { + if (slot == .none) continue; + if (slot_i != 0) try out_writer.writeAll(" | "); + switch (slot) { + .none => unreachable, + .reg => |reg| { + const reg_off = reg.index.offset(); + if (reg_off == 0) { + try out_writer.print("data[{}].enc()", .{slot_i}); + } else { + try out_writer.print("(@as(u32, data[{}].enc()) << {})", .{ slot_i, reg_off }); + } + }, + .imm => |imm| { + const imm_off = imm.index.offset(); + if (imm_off == 0) { + try out_writer.print("@as(u32, @intCast(@as(u{}, @bitCast(data[{}]))))", .{ imm.length, slot_i }); + } else { + try out_writer.print("(@as(u32, @as(u{}, @bitCast(data[{}]))) << {})", .{ imm.length, slot_i, imm_off }); + } + }, + } + } + try out_writer.print(",\n", .{}); + } + } + try out_writer.writeAll( + \\ }; + \\ } + \\}; + \\ + \\pub const Format = @import("std").meta.Tag(Data); + \\ + \\pub const Inst = struct { + \\ opcode: OpCode, + \\ data: Data, + \\ + ); + for (descs.items) |*desc| { + try out_writer.writeAll("\n"); + const name_buf = try arena.dupe(u8, desc.name); + defer arena.free(name_buf); + std.mem.replaceScalar(u8, name_buf, '.', '_'); + + try out_writer.writeAll(" // Workaround https://github.com/ziglang/zig/issues/24127\n"); + if (std.zig.isValidId(name_buf)) { + try out_writer.print(" pub noinline fn {s}(", .{name_buf}); + } else { + try out_writer.print(" pub noinline fn @\"{s}\"(", .{name_buf}); + } + for (desc.format, 0..) |slot, slot_i| { + if (slot == .none) continue; + if (slot_i != 0) try out_writer.writeAll(", "); + switch (slot) { + .none => unreachable, + .reg => |_| { + try out_writer.print("f{}: Register", .{slot_i}); + }, + .imm => |imm| { + try out_writer.print("f{}: {}", .{ slot_i, imm }); + }, + } + } + try out_writer.writeAll(") Inst {\n return .{ .opcode = "); + if (std.zig.isValidId(name_buf)) { + try out_writer.print(".{s}", .{name_buf}); + } else { + try out_writer.print(".@\"{s}\"", .{name_buf}); + } + + var format_buf = std.ArrayList(u8).init(arena); + defer format_buf.deinit(); + try OpcodeDesc.stringifyFormat(&desc.format, format_buf.writer()); + const format_str = format_buf.items; + + try out_writer.writeAll(", .data = "); + if (std.mem.eql(u8, format_str, "EMPTY")) { + try out_writer.writeAll(".EMPTY"); + } else { + try out_writer.print(".{{ .{s} = .{{", .{format_str}); + if (desc.format[1] != .none) try out_writer.writeAll(" "); + for (desc.format, 0..) |slot, slot_i| { + if (slot == .none) continue; + if (slot_i != 0) try out_writer.writeAll(", "); + try out_writer.print("f{}", .{slot_i}); + } + if (desc.format[1] != .none) try out_writer.writeAll(" "); + try out_writer.writeAll("} }"); + } + try out_writer.writeAll(" };\n }\n"); + } + try out_writer.writeAll( + \\}; + \\ + ); +} + +fn usageAndExit(arg0: []const u8, code: u8) noreturn { + const stderr = std.io.getStdErr(); + stderr.writer().print( + \\Usage: {s} /path/loongarch-opcodes /path/zig + \\ + \\Updates src/arch/loongarch/encoding.zig from https://github.com/loongson-community/loongarch-opcodes.git. + \\ + , .{arg0}) catch std.process.exit(1); + std.process.exit(code); +} + +const OpcodeDescList = std.ArrayList(OpcodeDesc); +const OpcodeFormatList = std.AutoHashMap(OpcodeDesc.Format, struct {}); + +const OpcodeDesc = struct { + word: u64, + name: []u8, + format: Format, + + pub const Format = [8]Slot; + + pub fn stringifyFormat(slots: *Format, writer: anytype) !void { + var all_none = true; + for (slots) |slot| { + if (slot == .none) continue; + all_none = false; + switch (slot) { + .none => unreachable, + .reg => |reg| { + switch (reg.index) { + .d => try writer.print("D", .{}), + .j => try writer.print("J", .{}), + .k => try writer.print("K", .{}), + .a => try writer.print("A", .{}), + else => try writer.print("{}", .{reg.index}), + } + }, + .imm => |imm| { + switch (imm.sign) { + .signed => try writer.print("S{}{}", .{ imm.index, imm.length }), + .unsigned => try writer.print("U{}{}", .{ imm.index, imm.length }), + } + }, + } + } + if (all_none) try writer.writeAll("EMPTY"); + } + + const Slot = union(enum) { + reg: struct { + class: enum { int, fp, fcc, lbt_scratch, lsx, lasx }, + index: Index, + }, + imm: struct { + sign: std.builtin.Signedness, + index: Index, + length: u5, + + pub fn format(imm: @This(), comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + try writer.print("{c}{}", .{ + @as(u8, switch (imm.sign) { + .signed => 'i', + .unsigned => 'u', + }), + imm.length, + }); + } + }, + none, + + pub fn format(ctx: Slot, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + switch (ctx) { + .none => try writer.print("none", .{}), + .reg => |reg| { + try writer.print("reg({s})@{}", .{ @tagName(reg.class), reg.index }); + }, + .imm => |imm| { + try writer.print("imm({s})@{}:{}", .{ @tagName(imm.sign), imm.index, imm.length }); + }, + } + } + + const Index = enum { + // zig fmt: off + d, j, k, a, m, n, + // zig fmt: on + + pub fn offset(index: Index) u5 { + return switch (index) { + .d => 0, + .j => 5, + .k => 10, + .a => 15, + .m => 16, + .n => 18, + }; + } + + pub fn format(ctx: Index, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + try writer.print("{s}", .{@tagName(ctx)}); + } + }; + }; +}; + +fn loadFromDescFile(list: *OpcodeDescList, allocator: Allocator, file: fs.File) !void { + const reader = file.reader(); + + var line_buf: [256]u8 = undefined; + while (try reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line_raw| { + try parseOpcodeDesc(list, allocator, line_raw); + } +} + +fn parseOpcodeDesc(list: *OpcodeDescList, allocator: Allocator, line_raw: []const u8) !void { + const line_trim = std.mem.trim(u8, line_raw, " \n"); + if (line_trim.len == 0) return; + var line_split = std.mem.splitScalar(u8, line_trim, ' '); + + const word_str = line_split.next() orelse unreachable; + const name_str = line_split.next() orelse return error.UnexpectedEol; + while (line_split.peek()) |part| { + if (part.len != 0) break; + _ = line_split.next(); + } + const format_str = line_split.next() orelse return error.UnexpectedEol; + + const word = try std.fmt.parseInt(u64, word_str, 16); + const name = try allocator.dupe(u8, name_str); + var desc = OpcodeDesc{ + .word = word, + .name = name, + .format = [1]OpcodeDesc.Slot{.none} ** 8, + }; + + print(" Instruction: {s} {s}\n", .{ name_str, format_str }); + var format_lexer = FormatLexer.init(format_str); + try format_lexer.parse(&desc); + for (&desc.format) |slot| { + if (slot == .none) continue; + print(" {}\n", .{slot}); + } + + try list.append(desc); +} + +const FormatLexer = struct { + buf: []const u8, + + pub fn init(desc: []const u8) FormatLexer { + return .{ .buf = desc }; + } + + pub fn parse(self: *FormatLexer, desc: *OpcodeDesc) !void { + if (std.mem.eql(u8, self.buf, "EMPTY")) return; + var slot_i: u3 = 0; + while (self.tryParse(FormatLexer.tryParseRegSlot)) |slot| { + desc.format[slot_i] = slot; + slot_i += 1; + } + while (self.tryParse(FormatLexer.tryParseImmSlots)) |slots| { + for (slots) |slot| { + if (slot == .none) continue; + desc.format[slot_i] = slot; + slot_i += 1; + } + } + if (self.buf.len != 0) return error.UnexpectedRemaining; + } + + fn nextChar(self: *FormatLexer) !u8 { + if (self.buf.len == 0) return error.UnexpectedEnd; + const ch = self.buf[0]; + self.buf = self.buf[1..]; + return ch; + } + + pub fn tryParse(self: *FormatLexer, comptime func: anytype) ?(@typeInfo(@typeInfo(@TypeOf(func)).@"fn".return_type.?).error_union.payload) { + const buf = self.buf; + if (func(self) catch null) |result| { + return result; + } else { + self.buf = buf; + return null; + } + } + + pub fn tryParseRegSlot(self: *FormatLexer) !OpcodeDesc.Slot { + switch (try self.nextChar()) { + 'D' => return .{ .reg = .{ .class = .int, .index = .d } }, + 'J' => return .{ .reg = .{ .class = .int, .index = .j } }, + 'K' => return .{ .reg = .{ .class = .int, .index = .k } }, + 'A' => return .{ .reg = .{ .class = .int, .index = .a } }, + 'F' => return .{ .reg = .{ .class = .fp, .index = try self.tryParseIndex() } }, + 'C' => return .{ .reg = .{ .class = .fcc, .index = try self.tryParseIndex() } }, + 'T' => return .{ .reg = .{ .class = .lbt_scratch, .index = try self.tryParseIndex() } }, + 'V' => return .{ .reg = .{ .class = .lsx, .index = try self.tryParseIndex() } }, + 'X' => return .{ .reg = .{ .class = .lasx, .index = try self.tryParseIndex() } }, + else => return error.UnknownRegSlot, + } + } + + pub fn tryParseImmSlots(self: *FormatLexer) ![8]OpcodeDesc.Slot { + const signedness = try self.tryParseSignedness(); + + var slots = [1]OpcodeDesc.Slot{.none} ** 8; + var slot_i: u3 = 0; + while (self.tryParse(FormatLexer.tryParseImmSlot)) |slot| { + slots[slot_i] = slot; + slots[slot_i].imm.sign = signedness; + slot_i += 1; + } + return slots; + } + + fn tryParseImmSlot(self: *FormatLexer) !OpcodeDesc.Slot { + const index = try self.tryParseIndex(); + const length = try self.tryParseInt(u5); + return .{ .imm = .{ .sign = undefined, .index = index, .length = length } }; + } + + pub fn tryParseIndex(self: *FormatLexer) !OpcodeDesc.Slot.Index { + switch (try self.nextChar()) { + 'd' => return .d, + 'j' => return .j, + 'k' => return .k, + 'a' => return .a, + 'm' => return .m, + 'n' => return .n, + else => return error.UnknownIndex, + } + } + + pub fn tryParseSignedness(self: *FormatLexer) !std.builtin.Signedness { + return switch (try self.nextChar()) { + 'S' => .signed, + 'U' => .unsigned, + else => return error.UnknownSignedness, + }; + } + + pub fn tryParseInt(self: *FormatLexer, comptime T: type) !T { + var len: usize = 0; + while (true) { + const ch = self.buf[len]; + if (ch < '0' or ch > '9') break; + len += 1; + if (len >= self.buf.len) break; + } + const str = self.buf[0..len]; + self.buf = self.buf[len..]; + return try std.fmt.parseInt(T, str, 10); + } +};