Convert a subset of GPU dialect ops to the OpenCL GPU runtime calls (#333)

* Convert a subset of GPU dialect ops to the GPU OpenCL runtime calls

Added a new path, that converts the following GPU dialect ops to the
corresponding callsof the GPU OpenCL runtime functions (to be
implemented later):
  - gpu.alloc, gpu.dealloc, gpu.memcpy and gpu.launch

The first argument of each runtime's function is a pointer to the
context structure. This is not a cl_context, this is an execution
context, i.e. a single execution of the module's main function. It
contains the queue, wait list (in case of out-of-order mode) and
someother data, required for the module ops execution. It's expected,
that the pointer to the context is passed to the module's main
function as the last argument of type memref with zero dims.

For each gpu.launch operation, 2 additional functions are created:
  - getXXXKernel(): returns the kernel pointer, stored in a global
    variable. If it's NULL, calls createXXXKernel().
  - createXXXKernel(): Calls the runtime's function, that creates a
    kernel. SPIRV, kernel name, and sizes are passed to the function.
    The returned pointer is saved in the global var using
    `llvm.cmpxchg`, to make sure it doesn't overwrite a kernel,
    created by another thread.

Finally, a destructor function is created, that calls the
corresponding runtime's kernel destroy function and passes the
pointers, stored in the global vars. This function must be called by
themodule owner, when destroying the module.

The kernel is not a cl_kernel, but a runtime's internal structure,
that contains a compiledcl_program, preconfigured cl_kernel and other
data, required for the kernel execution. The runtime's launch function
clones the preconfigured kernel, sets the arguments and enqueues a
command to execute the kernel.

Author: AndreyPavlenko

include/gc/ExecutionEngine/GPURuntime/GpuOclRuntime.h

@@ -0,0 +1,29 @@
+//===-- GpuOclRuntime.h - GPU OpenCL runtime --------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef GC_GPUOCLRUNTIME_H
+#define GC_GPUOCLRUNTIME_H
+
+namespace mlir::gc::gpu {
+constexpr char GPU_OCL_MALLOC[] = "gcGpuOclMalloc";
+constexpr char GPU_OCL_DEALLOC[] = "gcGpuOclDealloc";
+constexpr char GPU_OCL_MEMCPY[] = "gcGpuOclMemcpy";
+constexpr char GPU_OCL_KERNEL_CREATE[] = "gcGpuOclKernelCreate";
+constexpr char GPU_OCL_KERNEL_DESTROY[] = "gcGpuOclKernelDestroy";
+constexpr char GPU_OCL_KERNEL_LAUNCH[] = "gcGpuOclKernelLaunch";
+constexpr char GPU_OCL_MOD_DESTRUCTOR[] = "gcGpuOclModuleDestructor";
+} // namespace mlir::gc::gpu
+
+#ifndef GC_GPU_OCL_CONST_ONLY
+
+// TBD
+
+#else
+#undef GC_GPU_OCL_CONST_ONLY
+#endif
+#endif

include/gc/Transforms/Passes.td

@@ -93,6 +93,20 @@ def LinalgToXeGPU : Pass<"linalg-to-xegpu", "func::FuncOp"> {
                "DPAS register block sizes MxNxK">,
   ];
 }
+
+def AddContextArg : Pass<"add-ctx-arg", "func::FuncOp"> {
+  let summary = "Add a context argument.";
+  let description = [{
+    Add a new memref argument to the function, that could be used to pass some context.
+  }];
+}
+
+def GpuToGpuOcl : Pass<"gpu-to-gpuocl", "ModuleOp"> {
+  let summary = "Convert the GPU operations to GpuOclRuntime calls.";
+  let description = [{
+    Convert the gpu alloc, dealloc, memcpy and launch operations to GpuOclRuntime calls.
+  }];
+}
 #endif // GC_USE_IMEX
 
 def IterativeTilingAndFusion : Pass<"iterative-tiling-and-fusion",

lib/gc/Transforms/GPU/AddContextArg.cpp

@@ -0,0 +1,54 @@
+//===-- AddContextArg.cpp - Add context argument ----------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Conversion/Passes.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+
+namespace mlir::gc {
+#define GEN_PASS_DECL_ADDCONTEXTARG
+#define GEN_PASS_DEF_ADDCONTEXTARG
+#include "gc/Transforms/Passes.h.inc"
+} // namespace mlir::gc
+
+using namespace mlir;
+
+namespace {
+struct AddContextArg final : gc::impl::AddContextArgBase<AddContextArg> {
+  void runOnOperation() override {
+    auto func = getOperation();
+    if (func.isExternal()) {
+      return;
+    }
+
+    auto funcType = func.getFunctionType();
+    auto argTypes = llvm::to_vector<8>(funcType.getInputs());
+    auto resultTypes = llvm::to_vector<1>(funcType.getResults());
+    auto ctx = func->getContext();
+    auto newArgType = MemRefType::get({}, IntegerType::get(ctx, 8));
+    argTypes.emplace_back(newArgType);
+    auto newFuncType = FunctionType::get(ctx, argTypes, resultTypes);
+    func.setType(newFuncType);
+    func.getBody().front().addArgument(newArgType, func.getLoc());
+
+    // Find all function calls and append the last argument of the current
+    // function to the call.
+    auto module = func->getParentOfType<ModuleOp>();
+    func.walk([&](func::CallOp call) {
+      // If the function to be called is defined in the current module, then the
+      // context arg will be added to this function signature either and, thus,
+      // wee need add the context arg to the function call.
+      if (auto callee = module.lookupSymbol<func::FuncOp>(call.getCallee());
+          !callee || callee.isExternal()) {
+        return;
+      }
+      auto args = llvm::to_vector<8>(call.getOperands());
+      args.emplace_back(func.getArgument(func.getNumArguments() - 1));
+      call->setOperands(args);
+    });
+  }
+};
+} // namespace

lib/gc/Transforms/GPU/CMakeLists.txt

@@ -1,4 +1,6 @@
 gc_add_mlir_library(GcGpuPasses
+  AddContextArg.cpp
+  GpuToGpuOcl.cpp
   LinalgToXeGPU.cpp
   Pipeline.cpp