diff --git a/src/Update/PowerToys.Update.cpp b/src/Update/PowerToys.Update.cpp
index 31b6777bf0..f55bb0c161 100644
--- a/src/Update/PowerToys.Update.cpp
+++ b/src/Update/PowerToys.Update.cpp
@@ -169,14 +169,34 @@ bool InstallNewVersionStage2(std::wstring installer_path, std::wstring_view inst
         }
     }
 
+    for (const auto& entry : fs::directory_iterator(updating::get_pending_updates_path()))
+    {
+        auto entryPath = entry.path().wstring();
+        std::transform(entryPath.begin(), entryPath.end(), entryPath.begin(), ::towlower);
+
+        // Delete only .msi and .exe
+        if (entryPath.ends_with(L".msi") || entryPath.ends_with(L".exe"))
+        {
+            // Skipping current installer in case of failed update
+            if (installer_path.find(entryPath) != std::string::npos && !success)
+            {
+                continue;
+            }
+
+            std::error_code err;
+            fs::remove(entry, err);
+            if (err.value())
+            {
+                Logger::warn("Failed to delete file {}. {}", entry.path().string(), err.message());
+            }
+        }
+    }
+
     if (!success)
     {
         return false;
     }
 
-    std::error_code _;
-    fs::remove(installer_path, _);
-
     UpdateState::store([&](UpdateState& state) {
         state = {};
         state.githubUpdateLastCheckedDate.emplace(timeutil::now());