diff --git a/scripts/package/win-common.ps1 b/scripts/package/win-common.ps1
index 5add6e69c3..30e09fb500 100644
--- a/scripts/package/win-common.ps1
+++ b/scripts/package/win-common.ps1
@@ -41,9 +41,9 @@ Function FinalizePackage {
     $Path
   )
 
-  CodeSignBinary (Join-Path -Path $Path -ChildPath picard.exe)
-  CodeSignBinary (Join-Path -Path $Path -ChildPath fpcalc.exe)
-  CodeSignBinary (Join-Path -Path $Path -ChildPath discid.dll)
+  CodeSignBinary -BinaryPath (Join-Path -Path $Path -ChildPath picard.exe) -ErrorAction Stop
+  CodeSignBinary -BinaryPath (Join-Path -Path $Path -ChildPath fpcalc.exe) -ErrorAction Stop
+  CodeSignBinary -BinaryPath (Join-Path -Path $Path -ChildPath discid.dll) -ErrorAction Stop
 
   # Move all Qt5 DLLs into the main folder to avoid conflicts with system wide
   # versions of those dependencies. Since some version PyInstaller tries to
diff --git a/scripts/package/win-package-appx.ps1 b/scripts/package/win-package-appx.ps1
index a381223257..50f41f8664 100644
--- a/scripts/package/win-package-appx.ps1
+++ b/scripts/package/win-package-appx.ps1
@@ -12,6 +12,7 @@ Param(
 
 # Errors are handled explicitly. Otherwise any output to stderr when
 # calling classic Windows exes causes a script error.
+# TODO: For PowerShell >= 7.3 use $PSNativeCommandUseErrorActionPreference = $true
 $ErrorActionPreference = 'Continue'
 
 If (-Not $BuildNumber) {
@@ -68,4 +69,4 @@ If ($CertificateFile -or $Certificate) {
 MakeAppx pack /o /h SHA256 /d $PackageDir /p $PackageFile
 ThrowOnExeError "MakeAppx failed"
 
-CodeSignBinary $PackageFile
+CodeSignBinary -BinaryPath $PackageFile -ErrorAction Stop
diff --git a/scripts/package/win-package-installer.ps1 b/scripts/package/win-package-installer.ps1
index aa1d989cad..4653973df1 100644
--- a/scripts/package/win-package-installer.ps1
+++ b/scripts/package/win-package-installer.ps1
@@ -40,4 +40,4 @@ FinalizePackage dist\picard
 # Build the installer
 makensis.exe /INPUTCHARSET UTF8 installer\picard-setup.nsi 2>&1 | %{ "$_" }
 ThrowOnExeError "NSIS failed"
-CodeSignBinary installer\picard-setup-*.exe
+CodeSignBinary -BinaryPath installer\picard-setup-*.exe -ErrorAction Stop
diff --git a/scripts/package/win-package-portable.ps1 b/scripts/package/win-package-portable.ps1
index da5dc022ea..79b4633efc 100644
--- a/scripts/package/win-package-portable.ps1
+++ b/scripts/package/win-package-portable.ps1
@@ -36,4 +36,4 @@ ThrowOnExeError "setup.py build_ext -i failed"
 $env:PICARD_BUILD_PORTABLE = '1'
 pyinstaller --noconfirm --clean picard.spec 2>&1 | %{ "$_" }
 ThrowOnExeError "PyInstaller failed"
-CodeSignBinary dist\MusicBrainz-Picard-*.exe
+CodeSignBinary -BinaryPath dist\MusicBrainz-Picard-*.exe -ErrorAction Stop