From dd49735a22183d36003ced2206f8c990642d53b4 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 28 May 2025 10:43:09 +0800 Subject: [PATCH 1/5] enhance extra PRs apply --- .github/scripts/apply_torch_pr.py | 20 ++++++++------------ .github/workflows/nightly_ondemand_whl.yml | 3 +-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/scripts/apply_torch_pr.py b/.github/scripts/apply_torch_pr.py index 749614b71e..b44a193b85 100644 --- a/.github/scripts/apply_torch_pr.py +++ b/.github/scripts/apply_torch_pr.py @@ -55,22 +55,16 @@ def check_merged(pr_info): return merged def appyly_pr(pull_number, re_apply_msg): - # get the diff - os.system(f"\ - git fetch origin pull/{pull_number}/head:{pull_number} && \ - git checkout -f {pull_number} && \ - git merge ci-tmp-$(hostname) --no-edit --no-ff > /dev/null && \ - git diff ci-tmp-$(hostname) {pull_number} > {pull_number}.diff \ - ") # apply diff - os.system("git checkout ci-test-$(hostname)") - apply_cmd = f"git reset --hard && git apply --3way {pull_number}.diff" + apply_cmd = f"git reset --hard && git pull origin pull/{pull_number}/head --no-commit " apply_info = subprocess.Popen(apply_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) apply_message = apply_info.communicate()[0].decode("utf-8") apply_status = apply_info.returncode # apply status if apply_status == 0: print("{} {}, applied got SUCCESSFUL".format(pr_info["diff_url"], re_apply_msg)) + # get diff file + os.system(f"git restore --staged . && git diff > {pull_number}.diff") else: print("{} {}, applied got FAILED".format(pr_info["diff_url"], apply_message)) print(apply_status, apply_message) @@ -87,8 +81,7 @@ def appyly_pr(pull_number, re_apply_msg): pr_list = set(pr_list) pr_list = sorted(pr_list) # checkout a base branch -os.system("git checkout -b ci-tmp-$(hostname) && git checkout -b ci-test-$(hostname) && rm -f *.diff") -os.system("git config --global user.email intel.com && git config --global user.name intel") +need_apply = 0 for pr_link in pr_list: repo_info = pr_link.split("/") pr_info = requests.get('https://api.' + repo_info[-5] + '/repos/' + repo_info[-4] + '/' + \ @@ -106,6 +99,7 @@ def appyly_pr(pull_number, re_apply_msg): else: re_apply_msg = "is re-opened and reverted," appyly_pr(repo_info[-1], re_apply_msg) + need_apply += 1 elif pr_info["state"].lower() == "closed": merged_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Merged"), -1) re_apply_msg = "is closed but not merged" @@ -115,7 +109,9 @@ def appyly_pr(pull_number, re_apply_msg): print("{} is closed and merged, no need to apply".format(pr_info["diff_url"])) continue appyly_pr(repo_info[-1], re_apply_msg) + need_apply += 1 else: print("{} is {}, no need to apply".format(pr_info["diff_url"], pr_info["state"])) sys.exit(1) -os.system("git checkout ci-test-$(hostname) && git reset --hard && git apply --3way *.diff && git status") +if need_apply > 0: + os.system(f"git reset --hard && git apply --3way *.diff && git status") diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index fbee5dac3d..2bd80b3f88 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -116,7 +116,6 @@ jobs: cd pytorch && git checkout ${TORCH_COMMIT_ID} # apply PRs for stock pytorch pip install requests - # python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py git status && git show -s pip install -r requirements.txt TORCH_XPU_OPS_COMMIT=$( Date: Wed, 28 May 2025 23:09:19 +0800 Subject: [PATCH 2/5] get pr diff --- .github/scripts/apply_torch_pr.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/scripts/apply_torch_pr.py b/.github/scripts/apply_torch_pr.py index b44a193b85..edae86bc97 100644 --- a/.github/scripts/apply_torch_pr.py +++ b/.github/scripts/apply_torch_pr.py @@ -54,38 +54,33 @@ def check_merged(pr_info): merged = False return merged -def appyly_pr(pull_number, re_apply_msg): +def appyly_pr(pr_info, repo_info, re_apply_msg): + # get pr diff + pr_file = pr_info["diff_url"].split("/")[-1] + os.system(f"gh --repo {repo_info[-4]}/{repo_info[-3]} pr diff {repo_info[-1]} > {pr_file}") # apply diff - apply_cmd = f"git reset --hard && git pull origin pull/{pull_number}/head --no-commit " + apply_cmd = "git apply --3way " + pr_file apply_info = subprocess.Popen(apply_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) apply_message = apply_info.communicate()[0].decode("utf-8") apply_status = apply_info.returncode # apply status if apply_status == 0: print("{} {}, applied got SUCCESSFUL".format(pr_info["diff_url"], re_apply_msg)) - # get diff file - os.system(f"git restore --staged . && git diff > {pull_number}.diff") else: print("{} {}, applied got FAILED".format(pr_info["diff_url"], apply_message)) print(apply_status, apply_message) sys.exit(1) -github_token = os.environ.get("GH_TOKEN") -headers = { - "Authorization": f"Bearer {github_token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", -} +# headers = {'Authorization': 'Bearer ' + args.token} if args.token != None else args.token pr_list = args.pr_list + args.extra_pr_list pr_list = set(pr_list) pr_list = sorted(pr_list) -# checkout a base branch -need_apply = 0 for pr_link in pr_list: repo_info = pr_link.split("/") pr_info = requests.get('https://api.' + repo_info[-5] + '/repos/' + repo_info[-4] + '/' + \ - repo_info[-3] + '/pulls/' + repo_info[-1], headers=headers, timeout=60).json() + repo_info[-3] + '/pulls/' + repo_info[-1], timeout=60).json() + if pr_info["state"].lower() == "open": # for reverted PR reverted_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Reverted"), -1) @@ -98,8 +93,7 @@ def appyly_pr(pull_number, re_apply_msg): continue else: re_apply_msg = "is re-opened and reverted," - appyly_pr(repo_info[-1], re_apply_msg) - need_apply += 1 + appyly_pr(pr_info, repo_info, re_apply_msg) elif pr_info["state"].lower() == "closed": merged_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Merged"), -1) re_apply_msg = "is closed but not merged" @@ -108,10 +102,7 @@ def appyly_pr(pull_number, re_apply_msg): if merged: print("{} is closed and merged, no need to apply".format(pr_info["diff_url"])) continue - appyly_pr(repo_info[-1], re_apply_msg) - need_apply += 1 + appyly_pr(pr_info, repo_info, re_apply_msg) else: print("{} is {}, no need to apply".format(pr_info["diff_url"], pr_info["state"])) - sys.exit(1) -if need_apply > 0: - os.system(f"git reset --hard && git apply --3way *.diff && git status") + sys.exit(1) \ No newline at end of file From 0b9a2ab3396be682c825552bedb527bdbe783f70 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 28 May 2025 23:11:58 +0800 Subject: [PATCH 3/5] get pr diff --- .github/scripts/apply_torch_pr.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/scripts/apply_torch_pr.py b/.github/scripts/apply_torch_pr.py index edae86bc97..ef6ad0dd1e 100644 --- a/.github/scripts/apply_torch_pr.py +++ b/.github/scripts/apply_torch_pr.py @@ -72,14 +72,19 @@ def appyly_pr(pr_info, repo_info, re_apply_msg): sys.exit(1) -# headers = {'Authorization': 'Bearer ' + args.token} if args.token != None else args.token +github_token = os.environ.get("GH_TOKEN") +headers = { + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} pr_list = args.pr_list + args.extra_pr_list pr_list = set(pr_list) pr_list = sorted(pr_list) for pr_link in pr_list: repo_info = pr_link.split("/") pr_info = requests.get('https://api.' + repo_info[-5] + '/repos/' + repo_info[-4] + '/' + \ - repo_info[-3] + '/pulls/' + repo_info[-1], timeout=60).json() + repo_info[-3] + '/pulls/' + repo_info[-1], headers=headers, timeout=60).json() if pr_info["state"].lower() == "open": # for reverted PR From 62b4cd1e86d5867919c377bd84e03d596a169327 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 28 May 2025 23:13:20 +0800 Subject: [PATCH 4/5] update --- .github/scripts/apply_torch_pr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/apply_torch_pr.py b/.github/scripts/apply_torch_pr.py index ef6ad0dd1e..c027a820d1 100644 --- a/.github/scripts/apply_torch_pr.py +++ b/.github/scripts/apply_torch_pr.py @@ -85,7 +85,6 @@ def appyly_pr(pr_info, repo_info, re_apply_msg): repo_info = pr_link.split("/") pr_info = requests.get('https://api.' + repo_info[-5] + '/repos/' + repo_info[-4] + '/' + \ repo_info[-3] + '/pulls/' + repo_info[-1], headers=headers, timeout=60).json() - if pr_info["state"].lower() == "open": # for reverted PR reverted_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Reverted"), -1) @@ -110,4 +109,4 @@ def appyly_pr(pr_info, repo_info, re_apply_msg): appyly_pr(pr_info, repo_info, re_apply_msg) else: print("{} is {}, no need to apply".format(pr_info["diff_url"], pr_info["state"])) - sys.exit(1) \ No newline at end of file + sys.exit(1) From 35e4b9b355f052dde0aaedeef1ada1ffc1f604a3 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 29 May 2025 10:11:16 +0800 Subject: [PATCH 5/5] use latest apply method for ut --- .github/workflows/_linux_ut.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml index b2abfd1ec2..a94c49d0bd 100644 --- a/.github/workflows/_linux_ut.yml +++ b/.github/workflows/_linux_ut.yml @@ -119,7 +119,7 @@ jobs: cd third_party/torch-xpu-ops git checkout ${TORCH_XPU_OPS_COMMIT} cd ../.. - python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py + python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py fi pip install -r .ci/docker/requirements-ci.txt - name: Torch Config @@ -412,7 +412,7 @@ jobs: cd third_party/torch-xpu-ops git checkout ${TORCH_XPU_OPS_COMMIT} cd ../.. - python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py + python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py fi pip install -r .ci/docker/requirements-ci.txt - name: Torch Config