|
| 1 | +name: FastDeploy Linux GPU Build Task |
| 2 | +description: "FastDeploy packages build and upload" |
| 3 | + |
| 4 | +on: |
| 5 | + workflow_call: |
| 6 | + inputs: |
| 7 | + DOCKER_IMAGE: |
| 8 | + description: "Build Images" |
| 9 | + required: true |
| 10 | + type: string |
| 11 | + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310" |
| 12 | + FASTDEPLOY_ARCHIVE_URL: |
| 13 | + description: "URL of the compressed FastDeploy code archive." |
| 14 | + required: true |
| 15 | + type: string |
| 16 | + COMPILE_ARCH: |
| 17 | + description: "Build GPU Archs" |
| 18 | + required: true |
| 19 | + type: string |
| 20 | + default: "80,90" |
| 21 | + WITH_NIGHTLY_BUILD: |
| 22 | + description: "Enable nightly build mode (e.g. add date suffix to version)" |
| 23 | + required: false |
| 24 | + type: string |
| 25 | + default: "ON" |
| 26 | + FD_VERSION: |
| 27 | + description: "FastDeploy Package Version" |
| 28 | + required: false |
| 29 | + type: string |
| 30 | + default: "" |
| 31 | + UPLOAD: |
| 32 | + description: "Upload Package" |
| 33 | + required: false |
| 34 | + type: string |
| 35 | + default: "ON" |
| 36 | + CACHE_DIR: |
| 37 | + description: "Cache Dir Use" |
| 38 | + required: false |
| 39 | + type: string |
| 40 | + default: "" |
| 41 | + outputs: |
| 42 | + wheel_path: |
| 43 | + description: "Output path of the generated wheel" |
| 44 | + value: ${{ jobs.fd-build.outputs.wheel_path }} |
| 45 | +jobs: |
| 46 | + fd-build: |
| 47 | + runs-on: [self-hosted, GPU-Build] |
| 48 | + outputs: |
| 49 | + wheel_path: ${{ steps.set_output.outputs.wheel_path }} |
| 50 | + steps: |
| 51 | + - name: Code Prepare |
| 52 | + shell: bash |
| 53 | + env: |
| 54 | + docker_image: ${{ inputs.DOCKER_IMAGE }} |
| 55 | + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} |
| 56 | + IS_PR: ${{ github.event_name == 'pull_request' }} |
| 57 | + run: | |
| 58 | + set -x |
| 59 | + REPO="https://github.com/${{ github.repository }}.git" |
| 60 | + FULL_REPO="${{ github.repository }}" |
| 61 | + REPO_NAME="${FULL_REPO##*/}" |
| 62 | + BASE_BRANCH="${{ github.base_ref }}" |
| 63 | +
|
| 64 | + # Clean the repository directory before starting |
| 65 | + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ |
| 66 | + -e "REPO_NAME=${REPO_NAME}" \ |
| 67 | + ${docker_image} /bin/bash -c ' |
| 68 | + if [ -d ${REPO_NAME} ]; then |
| 69 | + echo "Directory ${REPO_NAME} exists, removing it..." |
| 70 | + rm -rf ${REPO_NAME}* |
| 71 | + fi |
| 72 | + ' |
| 73 | +
|
| 74 | + wget -q ${fd_archive_url} |
| 75 | + tar -xf FastDeploy.tar.gz |
| 76 | + rm -rf FastDeploy.tar.gz |
| 77 | + cd FastDeploy |
| 78 | + git config --global user.name "FastDeployCI" |
| 79 | + git config --global user.email "fastdeploy_ci@example.com" |
| 80 | + git log -n 3 --oneline |
| 81 | + - name: FastDeploy Build |
| 82 | + shell: bash |
| 83 | + env: |
| 84 | + docker_image: ${{ inputs.DOCKER_IMAGE }} |
| 85 | + compile_arch: ${{ inputs.COMPILE_ARCH }} |
| 86 | + fd_version: ${{ inputs.FD_VERSION }} |
| 87 | + CACHE_DIR: ${{ inputs.CACHE_DIR }} |
| 88 | + run: | |
| 89 | + set -x |
| 90 | + runner_name="${{ runner.name }}" |
| 91 | + CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}') |
| 92 | + gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,) |
| 93 | +
|
| 94 | + CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}" |
| 95 | + echo "CACHE_DIR is set to ${CACHE_DIR}" |
| 96 | + if [ ! -f "${CACHE_DIR}/gitconfig" ]; then |
| 97 | + touch "${CACHE_DIR}/gitconfig" |
| 98 | + fi |
| 99 | + PARENT_DIR=$(dirname "$WORKSPACE") |
| 100 | + echo "PARENT_DIR:$PARENT_DIR" |
| 101 | + docker run --rm --net=host \ |
| 102 | + --cap-add=SYS_PTRACE --privileged --shm-size=64G \ |
| 103 | + -v $(pwd):/workspace -w /workspace \ |
| 104 | + -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \ |
| 105 | + -v "${CACHE_DIR}/.cache:/root/.cache" \ |
| 106 | + -v "${CACHE_DIR}/ConfigDir:/root/.config" \ |
| 107 | + -e TZ="Asia/Shanghai" \ |
| 108 | + -e "COMPILE_ARCH=${compile_arch}" \ |
| 109 | + -e "FD_VERSION=${fd_version}" \ |
| 110 | + -e "WITH_NIGHTLY_BUILD=${WITH_NIGHTLY_BUILD}" \ |
| 111 | + --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c ' |
| 112 | + if [[ -n "${FD_VERSION}" ]]; then |
| 113 | + export FASTDEPLOY_VERSION=${FD_VERSION} |
| 114 | + echo "Custom FastDeploy version: ${FASTDEPLOY_VERSION}" |
| 115 | + fi |
| 116 | +
|
| 117 | + git config --global --add safe.directory /workspace/FastDeploy |
| 118 | + cd FastDeploy |
| 119 | + if [[ "${WITH_NIGHTLY_BUILD}" == "ON" ]];then |
| 120 | + GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD) |
| 121 | + DATE_ONLY=$(echo $GIT_COMMIT_TIME | sed "s/ .*//;s/-//g") |
| 122 | + echo "Git Commit Time: $GIT_COMMIT_TIME" |
| 123 | + echo "Date Only: $DATE_ONLY" |
| 124 | + export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}" |
| 125 | + fi |
| 126 | + pip config set global.index-url http://pip.baidu.com/root/baidu/+simple/ |
| 127 | + pip config set install.trusted-host pip.baidu.com |
| 128 | + pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple |
| 129 | +
|
| 130 | + python -m pip install --upgrade pip |
| 131 | + python -m pip install -r requirements.txt |
| 132 | + python -m pip install wheel |
| 133 | + python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ |
| 134 | + # 编译RDMA |
| 135 | + export ENABLE_FD_RDMA=1 |
| 136 | + bash build.sh 1 python false [${COMPILE_ARCH}] |
| 137 | + ls ./dist/*.whl |
| 138 | + ' |
| 139 | + - name: Package Upload |
| 140 | + id: set_output |
| 141 | + env: |
| 142 | + compile_arch: ${{ inputs.COMPILE_ARCH }} |
| 143 | + run: | |
| 144 | + set -x |
| 145 | + if [[ "${{ github.event_name }}" == "pull_request" ]];then |
| 146 | + commit_id=${{ github.event.pull_request.head.sha }} |
| 147 | + pr_num=${{ github.event.pull_request.number }} |
| 148 | + target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_} |
| 149 | + elif [[ "${{ github.ref_type }}" == "tag" ]]; then |
| 150 | + commit_id=${{ github.sha }} |
| 151 | + tag_name=${{ github.ref_name }} |
| 152 | + target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id}/SM${compile_arch//,/_} |
| 153 | + else |
| 154 | + commit_id=${{ github.sha }} |
| 155 | + branch_name=${{ github.ref_name }} |
| 156 | + target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id}/SM${compile_arch//,/_} |
| 157 | + fi |
| 158 | + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py |
| 159 | + push_file=$(realpath bos_tools.py) |
| 160 | + python --version |
| 161 | + python -m pip install bce-python-sdk==0.9.29 |
| 162 | + cd FastDeploy/dist/ |
| 163 | + matches=($(ls fastdeploy*.whl)) |
| 164 | + if [ ${#matches[@]} -ne 1 ]; then |
| 165 | + echo "Error: Found ${#matches[@]} matching files, expected exactly 1" |
| 166 | + exit 1 |
| 167 | + fi |
| 168 | + fd_wheel_name=${matches[0]} |
| 169 | + echo "Found: $fd_wheel_name" |
| 170 | + tree -L 3 |
| 171 | + python ${push_file} fastdeploy*.whl ${target_path} |
| 172 | + target_path_stripped="${target_path#paddle-github-action/}" |
| 173 | + WHEEL_PATH=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name} |
| 174 | + echo "wheel_path=${WHEEL_PATH}" >> $GITHUB_OUTPUT |
0 commit comments