diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index 20a29bf..ee3f80a 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -1,5 +1,11 @@ name: Build torch-mlir -description: Checkout torch-mlir repo and build it in mlir-test conda env +description: Checkout torch-mlir repo and build it in mlir conda env + +inputs: + conda_env: + required: true + type: string + description: Name of torch-mlir conda env runs: using: composite @@ -11,10 +17,56 @@ runs: path: torch-mlir ref: 'cpu-proto' submodules: recursive + + - name: Get Date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore Conda env cache + id: conda-cache + uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 2 + with: + path: ${{ env.CONDA }}/envs + key: >- + conda-${{ inputs.conda_env }}- + ${{ hashFiles(format('tests/conda-envs/{0}.yaml', inputs.conda_env)) }}- + ${{ steps.get-date.outputs.today }}- + ${{ env.CACHE_NUMBER }} + + - name: Update conda env for torch-mlir env + if: steps.conda-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/create_torch_mlir_conda_env + with: + conda_env: ${{ inputs.conda_env }} + - name: Build torch-mlir shell: bash -el {0} run: | + source ${CONDA}/bin/activate ${{ inputs.conda_env }} source /opt/intel/oneapi/setvars.sh - tests/scripts/build-mlir.sh torch-mlir + cd torch-mlir + cmake -GNinja -Bbuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython3_FIND_VIRTUALENV=ONLY \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DLLVM_TARGETS_TO_BUILD=host \ + externals/llvm-project/llvm + cmake --build build + cd .. + tar cf torch-mlir.tar torch-mlir echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV + + - name: Upload built mlir + uses: actions/upload-artifact@v3 + with: + name: torch-mlir-binary + path: torch-mlir.tar + if-no-files-found: error diff --git a/.github/actions/create_torch_mlir_conda_env/action.yml b/.github/actions/create_torch_mlir_conda_env/action.yml new file mode 100644 index 0000000..f7afe3e --- /dev/null +++ b/.github/actions/create_torch_mlir_conda_env/action.yml @@ -0,0 +1,19 @@ +name: Install torch-mlir required python packages +description: Checkout torch-mlir repo and install required packages in an empty conda env + +inputs: + conda_env: + required: true + type: string + description: Name of torch-mlir conda env + +runs: + using: composite + steps: + - name: Install required packages + shell: bash -el {0} + run: | + ${CONDA}/bin/conda env update --solver libmamba -n ${{ inputs.conda_env }} -f tests/conda-envs/${{ inputs.conda_env }}.yaml + source ${CONDA}/bin/activate ${{ inputs.conda_env }} + cd torch-mlir + pip install -r requirements.txt diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index d8259e9..897bd57 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -14,6 +14,17 @@ inputs: runs: using: composite steps: + - name: Download mlir that was built in previous job + if: ${{ inputs.conda_env == 'mlir' }} + uses: actions/download-artifact@v3 + with: + name: torch-mlir-binary + + - name: Setup benchmarks package + if: ${{ inputs.conda_env == 'mlir' }} + shell: bash -el {0} + run: tar xf torch-mlir.tar + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT @@ -33,16 +44,28 @@ runs: ${{ steps.get-date.outputs.today }}- ${{ env.CACHE_NUMBER }} - - name: Update conda env - if: steps.conda-cache.outputs.cache-hit != 'true' + - name: Update conda env for non-torch-mlir envs + if: steps.conda-cache.outputs.cache-hit != 'true' && inputs.conda_env != 'mlir' shell: bash -el {0} run: ${CONDA}/bin/conda env update --solver libmamba -n ${{ inputs.conda_env }} -f tests/conda-envs/${{ inputs.conda_env }}.yaml - - uses: ./.github/actions/build_mlir - if: ${{ inputs.conda_env == 'mlir' }} + - name: Update conda env for torch-mlir env + if: steps.conda-cache.outputs.cache-hit != 'true' && inputs.conda_env == 'mlir' + uses: ./.github/actions/create_torch_mlir_conda_env + with: + conda_env: ${{ inputs.conda_env }} - name: Setup benchmarks package shell: bash -el {0} run: | source ${CONDA}/bin/activate ${{ inputs.conda_env }} pip install -e . + + - name: Save environment variables for mlir env + if: ${{ inputs.conda_env == 'mlir' }} + shell: bash -el {0} + run: | + source ${CONDA}/bin/activate ${{ inputs.conda_env }} + source /opt/intel/oneapi/setvars.sh + echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index 7cea08a..a7bc497 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -20,10 +20,25 @@ on: required: true jobs: + build_mlir: + runs-on: + - self-hosted + - glados + steps: + - uses: actions/checkout@v4 + if: ${{ inputs.conda_env == 'mlir' }} + + - name: Build mlir and upload to artifact storage + if: ${{ inputs.conda_env == 'mlir' }} + uses: ./.github/actions/build_mlir + with: + conda_env: ${{ inputs.conda_env }} + mlp_test: runs-on: - self-hosted - glados + needs: build_mlir strategy: matrix: bname: @@ -54,6 +69,7 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/initial_setup with: conda_env: ${{ inputs.conda_env }} diff --git a/tests/scripts/build-mlir.sh b/tests/scripts/build-mlir.sh index 94c91ae..a1cdb8d 100755 --- a/tests/scripts/build-mlir.sh +++ b/tests/scripts/build-mlir.sh @@ -6,9 +6,6 @@ fi source ${CONDA}/bin/activate mlir -env -${CONDA}/bin/conda list - cd $1 pip install -r requirements.txt