[Bug]: guess_success for github resolver should consider more context in issue resolution
#5313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-Fix Tagged Issue with OpenHands | |
| on: | |
| workflow_call: | |
| inputs: | |
| max_iterations: | |
| required: false | |
| type: number | |
| default: 50 | |
| macro: | |
| required: false | |
| type: string | |
| default: "@openhands-agent" | |
| target_branch: | |
| required: false | |
| type: string | |
| default: "main" | |
| description: "Target branch to pull and create PR against" | |
| LLM_MODEL: | |
| required: false | |
| type: string | |
| default: "anthropic/claude-3-5-sonnet-20241022" | |
| base_container_image: | |
| required: false | |
| type: string | |
| default: "" | |
| description: "Custom sandbox env" | |
| secrets: | |
| LLM_MODEL: | |
| required: false | |
| LLM_API_KEY: | |
| required: true | |
| LLM_BASE_URL: | |
| required: false | |
| PAT_TOKEN: | |
| required: false | |
| PAT_USERNAME: | |
| required: false | |
| issues: | |
| types: [labeled] | |
| pull_request: | |
| types: [labeled] | |
| issue_comment: | |
| types: [created] | |
| pull_request_review_comment: | |
| types: [created] | |
| pull_request_review: | |
| types: [submitted] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| auto-fix: | |
| if: | | |
| github.event_name == 'workflow_call' || | |
| github.event.label.name == 'fix-me' || | |
| github.event.label.name == 'fix-me-experimental' || | |
| ( | |
| ((github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') && | |
| contains(github.event.comment.body, inputs.macro || '@openhands-agent') && | |
| (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER') | |
| ) || | |
| (github.event_name == 'pull_request_review' && | |
| contains(github.event.review.body, inputs.macro || '@openhands-agent') && | |
| (github.event.review.author_association == 'OWNER' || github.event.review.author_association == 'COLLABORATOR' || github.event.review.author_association == 'MEMBER') | |
| ) | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Get latest versions and create requirements.txt | |
| run: | | |
| python -m pip index versions openhands-ai > openhands_versions.txt | |
| OPENHANDS_VERSION=$(head -n 1 openhands_versions.txt | awk '{print $2}' | tr -d '()') | |
| echo "openhands-ai==${OPENHANDS_VERSION}" >> requirements.txt | |
| cat requirements.txt | |
| - name: Cache pip dependencies | |
| if: | | |
| !( | |
| github.event.label.name == 'fix-me-experimental' || | |
| ( | |
| (github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') && | |
| contains(github.event.comment.body, '@openhands-agent-exp') | |
| ) || | |
| ( | |
| github.event_name == 'pull_request_review' && | |
| contains(github.event.review.body, '@openhands-agent-exp') | |
| ) | |
| ) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/* | |
| key: ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }} | |
| - name: Check required environment variables | |
| env: | |
| LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} | |
| PAT_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| PAT_USERNAME: ${{ secrets.PAT_USERNAME }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| required_vars=("LLM_API_KEY") | |
| for var in "${required_vars[@]}"; do | |
| if [ -z "${!var}" ]; then | |
| echo "Error: Required environment variable $var is not set." | |
| exit 1 | |
| fi | |
| done | |
| # Check optional variables and warn about fallbacks | |
| if [ -z "$LLM_BASE_URL" ]; then | |
| echo "Warning: LLM_BASE_URL is not set, will use default API endpoint" | |
| fi | |
| if [ -z "$PAT_TOKEN" ]; then | |
| echo "Warning: PAT_TOKEN is not set, falling back to GITHUB_TOKEN" | |
| fi | |
| if [ -z "$PAT_USERNAME" ]; then | |
| echo "Warning: PAT_USERNAME is not set, will use openhands-agent" | |
| fi | |
| - name: Set environment variables | |
| run: | | |
| # Handle pull request events first | |
| if [ -n "${{ github.event.pull_request.number }}" ]; then | |
| echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| echo "ISSUE_TYPE=pr" >> $GITHUB_ENV | |
| # Handle pull request review events | |
| elif [ -n "${{ github.event.review.body }}" ]; then | |
| echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| echo "ISSUE_TYPE=pr" >> $GITHUB_ENV | |
| # Handle issue comment events that reference a PR | |
| elif [ -n "${{ github.event.issue.pull_request }}" ]; then | |
| echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV | |
| echo "ISSUE_TYPE=pr" >> $GITHUB_ENV | |
| # Handle regular issue events | |
| else | |
| echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV | |
| echo "ISSUE_TYPE=issue" >> $GITHUB_ENV | |
| fi | |
| if [ -n "${{ github.event.review.body }}" ]; then | |
| echo "COMMENT_ID=${{ github.event.review.id || 'None' }}" >> $GITHUB_ENV | |
| else | |
| echo "COMMENT_ID=${{ github.event.comment.id || 'None' }}" >> $GITHUB_ENV | |
| fi | |
| echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV | |
| echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.PAT_TOKEN || github.token }}" >> $GITHUB_ENV | |
| echo "SANDBOX_ENV_BASE_CONTAINER_IMAGE=${{ inputs.base_container_image }}" >> $GITHUB_ENV | |
| # Set branch variables | |
| echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV | |
| - name: Comment on issue with start message | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.PAT_TOKEN || github.token }} | |
| script: | | |
| const issueType = process.env.ISSUE_TYPE; | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ env.ISSUE_NUMBER }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` | |
| }); | |
| - name: Install OpenHands | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const commentBody = `${{ github.event.comment.body || '' }}`.trim(); | |
| const reviewBody = `${{ github.event.review.body || '' }}`.trim(); | |
| const labelName = `${{ github.event.label.name || '' }}`.trim(); | |
| const eventName = `${{ github.event_name }}`.trim(); | |
| // Check conditions | |
| const isExperimentalLabel = labelName === "fix-me-experimental"; | |
| const isIssueCommentExperimental = | |
| (eventName === "issue_comment" || eventName === "pull_request_review_comment") && | |
| commentBody.includes("@openhands-agent-exp"); | |
| const isReviewCommentExperimental = | |
| eventName === "pull_request_review" && reviewBody.includes("@openhands-agent-exp"); | |
| // Perform package installation | |
| if (isExperimentalLabel || isIssueCommentExperimental || isReviewCommentExperimental) { | |
| console.log("Installing experimental OpenHands..."); | |
| await exec.exec("python -m pip install --upgrade pip"); | |
| await exec.exec("pip install git+https://github.com/all-hands-ai/openhands.git"); | |
| } else { | |
| console.log("Installing from requirements.txt..."); | |
| await exec.exec("python -m pip install --upgrade pip"); | |
| await exec.exec("pip install -r requirements.txt"); | |
| } | |
| - name: Attempt to resolve issue | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }} | |
| GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }} | |
| LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} | |
| PYTHONPATH: "" | |
| run: | | |
| cd /tmp && python -m openhands.resolver.resolve_issue \ | |
| --repo ${{ github.repository }} \ | |
| --issue-number ${{ env.ISSUE_NUMBER }} \ | |
| --issue-type ${{ env.ISSUE_TYPE }} \ | |
| --max-iterations ${{ env.MAX_ITERATIONS }} \ | |
| --comment-id ${{ env.COMMENT_ID }} | |
| - name: Check resolution result | |
| id: check_result | |
| run: | | |
| if cd /tmp && grep -q '"success":true' output/output.jsonl; then | |
| echo "RESOLUTION_SUCCESS=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "RESOLUTION_SUCCESS=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Upload output.jsonl as artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() # Upload even if the previous steps fail | |
| with: | |
| name: resolver-output | |
| path: /tmp/output/output.jsonl | |
| retention-days: 30 # Keep the artifact for 30 days | |
| - name: Create draft PR or push branch | |
| if: always() # Create PR or branch even if the previous steps fail | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }} | |
| GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }} | |
| LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} | |
| PYTHONPATH: "" | |
| run: | | |
| if [ "${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}" == "true" ]; then | |
| cd /tmp && python -m openhands.resolver.send_pull_request \ | |
| --issue-number ${{ env.ISSUE_NUMBER }} \ | |
| --pr-type draft \ | |
| --reviewer ${{ github.actor }} | tee pr_result.txt && \ | |
| grep "draft created" pr_result.txt | sed 's/.*\///g' > pr_number.txt | |
| else | |
| cd /tmp && python -m openhands.resolver.send_pull_request \ | |
| --issue-number ${{ env.ISSUE_NUMBER }} \ | |
| --pr-type branch \ | |
| --send-on-failure | tee branch_result.txt && \ | |
| grep "branch created" branch_result.txt | sed 's/.*\///g; s/.expand=1//g' > branch_name.txt | |
| fi | |
| # Step leaves comment for when agent is invoked on PR | |
| - name: Analyze Push Logs (Updated PR or No Changes) # Skip comment if PR update was successful OR leave comment if the agent made no code changes | |
| uses: actions/github-script@v7 | |
| if: always() | |
| env: | |
| AGENT_RESPONDED: ${{ env.AGENT_RESPONDED || 'false' }} | |
| with: | |
| github-token: ${{ secrets.PAT_TOKEN || github.token }} | |
| script: | | |
| const fs = require('fs'); | |
| const issueNumber = ${{ env.ISSUE_NUMBER }}; | |
| let logContent = ''; | |
| try { | |
| logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim(); | |
| } catch (error) { | |
| console.error('Error reading pr_result.txt file:', error); | |
| } | |
| const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`; | |
| // Check logs from send_pull_request.py (pushes code to GitHub) | |
| if (logContent.includes("Updated pull request")) { | |
| console.log("Updated pull request found. Skipping comment."); | |
| process.env.AGENT_RESPONDED = 'true'; | |
| } else if (logContent.includes(noChangesMessage)) { | |
| github.rest.issues.createComment({ | |
| issue_number: issueNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.` | |
| }); | |
| process.env.AGENT_RESPONDED = 'true'; | |
| } | |
| # Step leaves comment for when agent is invoked on issue | |
| - name: Comment on issue # Comment link to either PR or branch created by agent | |
| uses: actions/github-script@v7 | |
| if: always() # Comment on issue even if the previous steps fail | |
| env: | |
| AGENT_RESPONDED: ${{ env.AGENT_RESPONDED || 'false' }} | |
| with: | |
| github-token: ${{ secrets.PAT_TOKEN || github.token }} | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const issueNumber = ${{ env.ISSUE_NUMBER }}; | |
| const success = ${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}; | |
| let prNumber = ''; | |
| let branchName = ''; | |
| let resultExplanation = ''; | |
| try { | |
| if (success) { | |
| prNumber = fs.readFileSync('/tmp/pr_number.txt', 'utf8').trim(); | |
| } else { | |
| branchName = fs.readFileSync('/tmp/branch_name.txt', 'utf8').trim(); | |
| } | |
| } catch (error) { | |
| console.error('Error reading file:', error); | |
| } | |
| try { | |
| if (!success){ | |
| // Read result_explanation from JSON file for failed resolution | |
| const outputFilePath = path.resolve('/tmp/output/output.jsonl'); | |
| if (fs.existsSync(outputFilePath)) { | |
| const outputContent = fs.readFileSync(outputFilePath, 'utf8'); | |
| const jsonLines = outputContent.split('\n').filter(line => line.trim() !== ''); | |
| if (jsonLines.length > 0) { | |
| // First entry in JSON lines has the key 'result_explanation' | |
| const firstEntry = JSON.parse(jsonLines[0]); | |
| resultExplanation = firstEntry.result_explanation || ''; | |
| } | |
| } | |
| } | |
| } catch (error){ | |
| console.error('Error reading file:', error); | |
| } | |
| // Check "success" log from resolver output | |
| if (success && prNumber) { | |
| github.rest.issues.createComment({ | |
| issue_number: issueNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `A potential fix has been generated and a draft PR #${prNumber} has been created. Please review the changes.` | |
| }); | |
| process.env.AGENT_RESPONDED = 'true'; | |
| } else if (!success && branchName) { | |
| let commentBody = `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`; | |
| if (resultExplanation) { | |
| commentBody += `\n\nAdditional details about the failure:\n${resultExplanation}`; | |
| } | |
| github.rest.issues.createComment({ | |
| issue_number: issueNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: commentBody | |
| }); | |
| process.env.AGENT_RESPONDED = 'true'; | |
| } | |
| # Leave error comment when both PR/Issue comment handling fail | |
| - name: Fallback Error Comment | |
| uses: actions/github-script@v7 | |
| if: ${{ env.AGENT_RESPONDED == 'false' }} # Only run if no conditions were met in previous steps | |
| with: | |
| github-token: ${{ secrets.PAT_TOKEN || github.token }} | |
| script: | | |
| const issueNumber = ${{ env.ISSUE_NUMBER }}; | |
| github.rest.issues.createComment({ | |
| issue_number: issueNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.` | |
| }); |