Skip to content

feat: Validate PR action posts results as PR comment, passing-checks label, and auto-update outdated branches #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amanintech merged 4 commits into from
Mar 29, 2026
Merged

feat: Validate PR action posts results as PR comment, passing-checks label, and auto-update outdated branches #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cf14508
Initial plan
Copilot Mar 29, 2026
8b9b313
wip: plan for passing-checks label
Copilot Mar 29, 2026
96fb6d2
wip: plan for outdated branch update
Copilot Mar 29, 2026
bffa0c8
feat: update outdated branch, passing-checks label, PR comments, link…
Copilot Mar 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 123 additions & 6 deletions .github/workflows/review-challenge-prs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ jobs:
permissions:
pull-requests: write
issues: write
contents: write
steps:
- name: Review agentkit-challenge PRs
uses: actions/github-script@v7
Expand All @@ -30,6 +31,7 @@ jobs:
: false;
const LABEL = 'agentkit-challenge';
const CODERABBIT_LOGIN = 'coderabbitai[bot]';
const LABEL_PASSING = 'passing-checks';

// Unique markers embedded in bot comments to prevent duplicates
const MARKER_CHANGES = '<!-- agentkit-review-bot: changes-requested -->';
Expand Down Expand Up @@ -68,18 +70,121 @@ jobs:

console.log(`Found ${challengePRs.length} open PRs with label '${LABEL}'`);

// Ensure the passing-checks label exists in the repo
if (!DRY_RUN) {
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: LABEL_PASSING,
});
} catch (e) {
if (e.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: LABEL_PASSING,
color: '0e8a16',
description: 'All validation checks passed',
});
console.log(`Created label '${LABEL_PASSING}'`);
}
}
}

const addPassingLabel = async (prNum) => {
if (DRY_RUN) {
console.log(` [DRY RUN] Would add '${LABEL_PASSING}' label`);
return;
}
try {
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNum,
labels: [LABEL_PASSING],
});
console.log(` ✓ Added '${LABEL_PASSING}' label`);
} catch (e) {
console.log(` Warning: Could not add label: ${e.message}`);
}
};

const removePassingLabel = async (prNum) => {
if (DRY_RUN) {
console.log(` [DRY RUN] Would remove '${LABEL_PASSING}' label`);
return;
}
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNum,
name: LABEL_PASSING,
});
console.log(` ✓ Removed '${LABEL_PASSING}' label`);
} catch (e) {
if (e.status !== 404) console.log(` Warning: Could not remove label: ${e.message}`);
}
};

const summary = {
noReview: [],
changesRequested: [],
actionableComments: [],
clean: [],
branchUpdated: [],
};

for (const pr of challengePRs) {
const prNum = pr.number;
const author = pr.user.login;
console.log(`\nChecking PR #${prNum}: "${pr.title}" by @${author}`);

// Update branch if it is behind the base branch.
// Possible mergeable_state values and how they are handled:
// 'behind' — branch needs base commits merged in → update automatically (handled here)
// 'clean' — up to date and ready to merge → no action needed
// 'dirty' — has merge conflicts → requires author intervention, skip
// 'unstable' — CI is failing → not a branch-staleness issue, skip
// 'blocked' — blocked by branch protection rules → skip
// 'unknown' — GitHub is still computing mergeability → skip, retry on next run
// null — PR details fetch failed → logged and skipped
let mergeableState = null;
try {
const prDetails = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: prNum,
});
mergeableState = prDetails.data.mergeable_state;
} catch (e) {
console.log(` Warning: Could not fetch PR details to check branch status: ${e.message}`);
}

if (mergeableState === 'behind') {
console.log(` → Branch is behind base branch. Updating.`);
summary.branchUpdated.push(`[#${prNum}](${pr.html_url}) by @${author} — "${pr.title}"`);
if (!DRY_RUN) {
try {
await github.rest.pulls.updateBranch({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: prNum,
});
console.log(` ✓ Branch updated`);
} catch (e) {
console.log(` Warning: Could not update branch: ${e.message}`);
}
} else {
console.log(` [DRY RUN] Would update branch`);
}
} else if (mergeableState === null) {
console.log(` → Branch status unknown (PR details fetch failed). Skipping update.`);
} else {
console.log(` → Branch state is '${mergeableState}'. No automatic update needed.`);
}

// Get all reviews for this PR
const reviewsResp = await github.rest.pulls.listReviews({
owner: context.repo.owner,
Expand All @@ -92,7 +197,7 @@ jobs:

if (codeRabbitReviews.length === 0) {
console.log(` → No CodeRabbit review found. Triggering review.`);
summary.noReview.push(`#${prNum} by @${author} — "${pr.title}"`);
summary.noReview.push(`[#${prNum}](${pr.html_url}) by @${author} — "${pr.title}"`);
if (!DRY_RUN) {
await github.rest.issues.createComment({
owner: context.repo.owner,
Expand All @@ -104,6 +209,7 @@ jobs:
} else {
console.log(` [DRY RUN] Would trigger CodeRabbit review`);
}
await removePassingLabel(prNum);
continue;
}

Expand Down Expand Up @@ -142,7 +248,7 @@ jobs:

if (isStale) {
console.log(` → CodeRabbit review is stale (new commits after last review). Triggering re-review.`);
summary.noReview.push(`#${prNum} by @${author} — stale review, re-triggered`);
summary.noReview.push(`[#${prNum}](${pr.html_url}) by @${author} — stale review, re-triggered`);
if (!DRY_RUN) {
await github.rest.issues.createComment({
owner: context.repo.owner,
Expand All @@ -154,12 +260,13 @@ jobs:
} else {
console.log(` [DRY RUN] Would trigger CodeRabbit re-review`);
}
await removePassingLabel(prNum);
continue;
}

if (hasChangesRequested) {
console.log(` → Has CHANGES_REQUESTED from CodeRabbit. Asking author to resolve.`);
summary.changesRequested.push(`#${prNum} by @${author} — "${pr.title}"`);
summary.changesRequested.push(`[#${prNum}](${pr.html_url}) by @${author} — "${pr.title}"`);

// Check if we already posted this message recently (avoid spam)
const commentsResp = await github.rest.issues.listComments({
Expand Down Expand Up @@ -187,6 +294,7 @@ jobs:
} else {
console.log(` [DRY RUN] Would post reminder to resolve CodeRabbit changes`);
}
await removePassingLabel(prNum);
} else if (codeRabbitReviews.some(r =>
r.state === 'COMMENTED' && r.body && r.body.includes('Actionable comments posted:')
)) {
Expand All @@ -200,7 +308,7 @@ jobs:
if (actionableCount > 0) {
console.log(` → Has ${actionableCount} actionable comments. Asking author to address them.`);
summary.actionableComments.push(
`#${prNum} by @${author} — ${actionableCount} actionable comments — "${pr.title}"`
`[#${prNum}](${pr.html_url}) by @${author} — ${actionableCount} actionable comments — "${pr.title}"`
);

const commentsResp = await github.rest.issues.listComments({
Expand Down Expand Up @@ -228,13 +336,16 @@ jobs:
} else {
console.log(` [DRY RUN] Would post reminder to address CodeRabbit comments`);
}
await removePassingLabel(prNum);
} else {
console.log(` → CodeRabbit reviewed with no blocking comments. Ready for maintainer review.`);
summary.clean.push(`#${prNum} by @${author} — "${pr.title}"`);
summary.clean.push(`[#${prNum}](${pr.html_url}) by @${author} — "${pr.title}"`);
await addPassingLabel(prNum);
}
} else {
console.log(` → No actionable CodeRabbit comments. May be ready for review.`);
summary.clean.push(`#${prNum} by @${author} — "${pr.title}"`);
summary.clean.push(`[#${prNum}](${pr.html_url}) by @${author} — "${pr.title}"`);
await addPassingLabel(prNum);
}
}

Expand All @@ -243,6 +354,12 @@ jobs:
.addHeading('AgentKit Challenge PR Review Summary', 2)
.addRaw(`\n**Total PRs checked:** ${challengePRs.length}\n\n`);

if (summary.branchUpdated.length > 0) {
core.summary
.addHeading('🔀 Branch Updated (was behind base)', 3)
.addList(summary.branchUpdated);
}

if (summary.noReview.length > 0) {
core.summary
.addHeading('🔄 CodeRabbit Review Triggered (no/stale review)', 3)
Expand Down
78 changes: 75 additions & 3 deletions .github/workflows/validate-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ jobs:
if: startsWith(github.event.pull_request.title, 'feat:')
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
issues: write
steps:
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -19,6 +23,7 @@ jobs:
run: git fetch origin ${{ github.event.pull_request.base.ref }}

- name: Validate contribution structure
id: validate
env:
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
Expand All @@ -36,8 +41,13 @@ jobs:

if [ -z "$CHANGED_FILES" ]; then
echo "No changed files detected."
echo "## PR Validation Results" >> "$GITHUB_STEP_SUMMARY"
echo "No contribution files detected in this PR." >> "$GITHUB_STEP_SUMMARY"
SUMMARY_FILE="/tmp/pr_validation_summary.md"
{
echo "## PR Validation Results"
echo ""
echo "No contribution files detected in this PR."
} > "$SUMMARY_FILE"
cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi

Expand Down Expand Up @@ -224,6 +234,7 @@ jobs:

# --- F. Output results to job summary ---

SUMMARY_FILE="/tmp/pr_validation_summary.md"
{
echo "## PR Validation Results"
echo ""
Expand Down Expand Up @@ -303,7 +314,9 @@ jobs:
echo ""
echo "Refer to [CONTRIBUTING.md](./CONTRIBUTING.md) and [CLAUDE.md](./CLAUDE.md) for the expected folder structure."
fi
} >> "$GITHUB_STEP_SUMMARY"
} > "$SUMMARY_FILE"

cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"

# Print to logs as well
if [ ${#ERRORS[@]} -gt 0 ]; then
Expand All @@ -317,3 +330,62 @@ jobs:

echo ""
echo "=== ALL CHECKS PASSED ==="

- name: Post validation results as PR comment
if: always()
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
SUMMARY_FILE="/tmp/pr_validation_summary.md"
PR_NUMBER="${{ github.event.pull_request.number }}"
REPO="${{ github.repository }}"

if [ ! -f "$SUMMARY_FILE" ]; then
echo "No summary file found, skipping comment."
exit 0
fi

# Find an existing bot comment that starts with the validation header
COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
--jq '.[] | select(.user.type == "Bot" and (.body | startswith("## PR Validation Results"))) | .id' \
| head -1)

if [ -n "$COMMENT_ID" ]; then
gh api "repos/$REPO/issues/comments/$COMMENT_ID" \
--method PATCH \
-f body="$(cat "$SUMMARY_FILE")"
else
gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
--method POST \
-f body="$(cat "$SUMMARY_FILE")"
fi

- name: Apply passing-checks label
if: always()
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PR_NUMBER="${{ github.event.pull_request.number }}"
REPO="${{ github.repository }}"
LABEL="passing-checks"

# Ensure the label exists in the repo
if ! gh api "repos/$REPO/labels/$LABEL" --silent 2>/dev/null; then
if gh api "repos/$REPO/labels" \
--method POST \
-f name="$LABEL" \
-f color="0e8a16" \
-f description="All validation checks passed" 2>/dev/null; then
echo "Created label '$LABEL'"
else
echo "Warning: Could not create label '$LABEL' (insufficient permissions or network error)"
fi
fi

if [ "${{ steps.validate.outcome }}" = "success" ]; then
gh pr edit "$PR_NUMBER" --add-label "$LABEL" --repo "$REPO"
echo "Added '$LABEL' label to PR #$PR_NUMBER"
else
gh pr edit "$PR_NUMBER" --remove-label "$LABEL" --repo "$REPO" 2>/dev/null || true
echo "Removed '$LABEL' label from PR #$PR_NUMBER (validation failed)"
fi
Loading