Test HSA runtime launcher on real hardware

[Zaneham]

The new HSA runtime launcher (src/runtime/bc_runtime.c) needs testing on a Linux machine with ROCm and an AMD GPU. Build instructions in the README under "Runtime Launcher".

Any RDNA 2/3/4 GPU should work. Looking for a PASS: all 1024 elements correct from launch_saxpy.

[statusfailed]

Failed for me:

+ gcc -std=c99 -O2 -I src/runtime examples/launch_saxpy.c src/runtime/bc_runtime.c -ldl -lm -o launch_saxpy
+ ./barracuda --amdgpu-bin -o test.hsaco tests/canonical.cu
wrote test.hsaco (9136 bytes code, 8 kernels)
+ ./launch_saxpy test.hsaco
BarraCUDA SAXPY launcher
  .hsaco:  test.hsaco
  N:       1024
  a:       2.5
bc_runtime: GPU ready, queue size 131072
bc_runtime: executable_load failed (0x1010)
Kernel load failed (-3)

rocminfo Name: AMD RYZEN AI MAX+ PRO 395 w/ Radeon 8060S

Tested in NixOS with this flake.nix:

{
  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

  outputs = { nixpkgs, ... }:
    let
      forAll = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
    in
    {
      devShells = forAll (system:
        let
          pkgs = import nixpkgs { inherit system; };
          hsa = pkgs.rocmPackages.rocm-runtime;
        in
        {
          default = pkgs.mkShell {
            packages = [ pkgs.gcc pkgs.gnumake hsa ];
            shellHook = ''
              export LD_LIBRARY_PATH=${hsa}/lib:${hsa}/lib64''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
            '';
          };
        });
    };
}

nixos-unstable's ROCm seems to work for ollama and rocblas-test

[Zaneham]

Thanks for testing! Your GPU is gfx1151 - we were stamping gfx1100 in the ELF, so HSA rejects it. Same ISA, wrong metadata.

Fixed on fixesAMD:

./barracuda --amdgpu-bin --gfx1151 -o test.hsaco tests/canonical.cu
./launch_saxpy test.hsaco

Let me know if it passes.

Edit: thanks again so much for testing this!! :-)

[Zaneham]

Woopsy thats a blip on my end, I'll create a separate issue.

[statusfailed]

Tried this again on 171f238 but I'm still seeing the same error

[statusfailed]

Gave this to codex and received the patch below which seems to fix the hsaco loading issue. Changes summary:

• Switched ELF type to ET_DYN for code objects.
• Added program headers (PT_LOAD for .text, PT_NOTE for metadata note).
• Set valid program-header table fields (e_phoff, e_phentsize, e_phnum).
• Set AMDGPU HSA ABI version byte (e_ident[EI_ABIVERSION] = 1, HSA V3-compatible).

Not submitting this as a PR, because I assume the e_ident[EI_ABIVERSION] = 1 line will break other platforms.

After running with this patch, I've unlocked a new, more exciting error:

BarraCUDA SAXPY launcher
  .hsaco:  test.hsaco
  N:       1024
  a:       2.5
bc_runtime: GPU ready, queue size 131072
bc_runtime: loaded 'saxpy' (kernarg=32, lds=0, scratch=0)
  dispatch: 4 blocks x 256 threads
Queue at 0x7ffff7b96000 inactivated due to async error:
        HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION: The agent attempted to access memory beyond the largest legal address.
./run.sh: line 15: 678661 Aborted                    (core dumped) nix develop .#default --command ./launch_saxpy test.hsaco

diff --git a/src/amdgpu/emit.c b/src/amdgpu/emit.c
index 3933f8f..29ffbcc 100644
--- a/src/amdgpu/emit.c
+++ b/src/amdgpu/emit.c
@@ -787,6 +787,17 @@ typedef struct {
     uint64_t sh_entsize;
 } elf64_shdr_t;   /* 64 bytes */
 
+typedef struct {
+    uint32_t p_type;
+    uint32_t p_flags;
+    uint64_t p_offset;
+    uint64_t p_vaddr;
+    uint64_t p_paddr;
+    uint64_t p_filesz;
+    uint64_t p_memsz;
+    uint64_t p_align;
+} elf64_phdr_t;   /* 56 bytes */
+
 typedef struct {
     uint32_t st_name;
     uint8_t  st_info;
@@ -807,6 +818,11 @@ typedef struct {
 #define SHT_SYMTAB   2
 #define SHT_STRTAB   3
 #define SHT_NOTE     7
+#define PT_LOAD      1
+#define PT_NOTE      4
+#define PF_X         1
+#define PF_W         2
+#define PF_R         4
 #define SHF_ALLOC    2
 #define SHF_EXECINSTR 4
 #define STB_GLOBAL   1
@@ -1075,7 +1091,9 @@ int amdgpu_emit_elf(amd_module_t *A, const char *path)
     }
 
     /* Compute layout */
-    uint64_t text_off = 64; /* after ELF header */
+    const uint16_t num_phdrs = 2;
+    uint64_t phdr_off = 64; /* right after ELF header */
+    uint64_t text_off = (phdr_off + (uint64_t)num_phdrs * 56 + 255) & ~255ULL;
     uint64_t text_size = A->code_len;
     uint64_t note_off = text_off + text_size;
     /* Align note to 4 bytes */
@@ -1101,21 +1119,40 @@ int amdgpu_emit_elf(amd_module_t *A, const char *path)
     ehdr.e_ident[5] = 1;    /* ELFDATA2LSB */
     ehdr.e_ident[6] = 1;    /* EV_CURRENT */
     ehdr.e_ident[7] = ELFOSABI_AMDGPU_HSA;
-    ehdr.e_type = 2;        /* ET_EXEC (shared object would be 3) */
+    ehdr.e_ident[8] = 1;    /* ELFABIVERSION_AMDGPU_HSA_V3 */
+    ehdr.e_type = 3;        /* ET_DYN (AMDGPU code object) */
     ehdr.e_machine = EM_AMDGPU;
     ehdr.e_version = 1;
     ehdr.e_entry = 0;
-    ehdr.e_phoff = 0;
+    ehdr.e_phoff = phdr_off;
     ehdr.e_shoff = shdr_off;
     ehdr.e_flags = A->elf_mach;
     ehdr.e_ehsize = 64;
-    ehdr.e_phentsize = 0;
-    ehdr.e_phnum = 0;
+    ehdr.e_phentsize = 56;
+    ehdr.e_phnum = num_phdrs;
     ehdr.e_shentsize = 64;
     ehdr.e_shnum = num_sections;
     ehdr.e_shstrndx = 5;    /* .shstrtab index */
     fwrite(&ehdr, 1, 64, fp);
 
+    /* Program headers: one RX load segment and one note segment. */
+    elf64_phdr_t phdrs[2];
+    memset(phdrs, 0, sizeof(phdrs));
+    phdrs[0].p_type = PT_LOAD;
+    phdrs[0].p_flags = PF_R | PF_X;
+    phdrs[0].p_offset = text_off;
+    phdrs[0].p_filesz = text_size;
+    phdrs[0].p_memsz = text_size;
+    phdrs[0].p_align = 256;
+    phdrs[1].p_type = PT_NOTE;
+    phdrs[1].p_flags = PF_R;
+    phdrs[1].p_offset = note_off;
+    phdrs[1].p_filesz = note_len;
+    phdrs[1].p_memsz = note_len;
+    phdrs[1].p_align = 4;
+    fwrite(phdrs, sizeof(elf64_phdr_t), num_phdrs, fp);
+    fwrite_pad(fp, 256);
+
     /* .text section */
     fwrite(A->code, 1, A->code_len, fp);
     fwrite_pad(fp, 4);