diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 12ba9989..45cd6841 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -95,7 +95,7 @@ jobs:
     - name: Upload Provider Binaries
       run: aws s3 cp dist s3://get.pulumi.com/releases/plugins/ --recursive
     - name: Create GH Release
-      uses: softprops/action-gh-release@v1
+      uses: softprops/action-gh-release@v2
       if: inputs.isPrerelease == false
       with:
         tag_name: v${{ inputs.version }}
diff --git a/.github/workflows/upgrade-provider.yml b/.github/workflows/upgrade-provider.yml
index 22386fa7..7d0056bc 100644
--- a/.github/workflows/upgrade-provider.yml
+++ b/.github/workflows/upgrade-provider.yml
@@ -7,7 +7,7 @@ on:
       version:
         description: |
           The version of the upstream provider to upgrade to, without the 'v' prefix
-          
+
           If no version is specified, it will be inferred from the upstream provider's release tags.
         required: false
         type: string
@@ -55,7 +55,16 @@ jobs:
         run: echo "version=${{ github.event.inputs.version || steps.upstream_version.outputs.latest_version }}" >> "$GITHUB_OUTPUT"
         shell: bash
       - name: Attempt provider upgrade
+        id: upgrade_provider
         # Only attempt the upgrade if we have a target version
         if: steps.target_version.outputs.version != ''
+        # Don't mark the build as failed if we can't auto-open a PR as we've already opened the upgrade issue for tracking
+        continue-on-error: true
         run: upgrade-provider "${{ github.repository }}" --kind="all" --target-version="${{ steps.target_version.outputs.version }}" 
         shell: bash
+      - name: Comment on upgrade issue if automated PR failed
+        if: steps.upgrade_provider.outcome == 'failure'
+        shell: bash
+        run: |
+          issue_number=$(gh issue list --search "pulumiupgradeproviderissue" --repo "${{ github.repository }}" --json=number --jq=".[0].number")
+          gh issue comment "${issue_number}" --repo "${{ github.repository }}" --body "Failed to create automatic PR: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/"
diff --git a/Makefile b/Makefile
index a2c17eb7..4619eb01 100644
--- a/Makefile
+++ b/Makefile
@@ -181,7 +181,6 @@ bin/pulumi-java-gen: .pulumi-java-gen.version
 # - Run make ci-mgmt to apply the change locally.
 #
 ci-mgmt: .ci-mgmt.yaml
-	find .github/workflows/*.yml -type f ! -name "$(PACK)*.yml" -delete
 	go run github.com/pulumi/ci-mgmt/provider-ci@master generate \
 		--name $(ORG)/pulumi-$(PACK) \
 		--out . \