diff --git a/RELEASES.md b/RELEASES.md
index 7fe4dac1..5b529f81 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -11,6 +11,7 @@
 ### Usability
 
 - Implemented `Eq` for `Timing` and `InputMap`.
+- Held `ActionState` inputs will now be released when an `InputMap` is removed.
 
 ## Version 0.5
 
diff --git a/src/plugin.rs b/src/plugin.rs
index 269e8107..c4180379 100644
--- a/src/plugin.rs
+++ b/src/plugin.rs
@@ -105,7 +105,8 @@ impl<A: Actionlike> Plugin for InputManagerPlugin<A> {
                     release_on_disable::<A>
                         .label(InputManagerSystem::ReleaseOnDisable)
                         .after(InputManagerSystem::Update),
-                );
+                )
+                .add_system_to_stage(CoreStage::PostUpdate, release_on_input_map_removed::<A>);
 
                 #[cfg(feature = "ui")]
                 app.add_system_to_stage(
diff --git a/src/systems.rs b/src/systems.rs
index 430b17fb..efa2f812 100644
--- a/src/systems.rs
+++ b/src/systems.rs
@@ -212,6 +212,39 @@ pub fn release_on_disable<A: Actionlike>(
     }
 }
 
+/// Release all inputs when an [`InputMap<A>`] is removed to prevent them from being held forever.
+///
+/// By default, [`InputManagerPlugin<A>`] will run this on [`CoreStage::PostUpdate`](bevy::prelude::CoreStage::PostUpdate).
+/// For components you must remove the [`InputMap<A>`] before [`CoreStage::PostUpdate`](bevy::prelude::CoreStage::PostUpdate)
+/// or this will not run.
+pub fn release_on_input_map_removed<A: Actionlike>(
+    removed_components: RemovedComponents<InputMap<A>>,
+    input_map_resource: Option<ResMut<InputMap<A>>>,
+    action_state_resource: Option<ResMut<ActionState<A>>>,
+    mut input_map_resource_existed: Local<bool>,
+    mut action_state_query: Query<&mut ActionState<A>>,
+) {
+    let mut iter = action_state_query.iter_many_mut(removed_components.iter());
+    while let Some(mut action_state) = iter.fetch_next() {
+        action_state.release_all();
+    }
+
+    // Detect when an InputMap resource is removed.
+    if input_map_resource.is_some() {
+        // Store if the resource existed so we know if it was removed later.
+        *input_map_resource_existed = true;
+    } else if *input_map_resource_existed {
+        // The input map does not exist and our local is true so we know the input map was removed.
+
+        if let Some(mut action_state) = action_state_resource {
+            action_state.release_all();
+        }
+
+        // Reset our local so our removal detection is only triggered once.
+        *input_map_resource_existed = false;
+    }
+}
+
 /// Returns [`ShouldRun::No`] if [`DisableInput`] exists and [`ShouldRun::Yes`] otherwise
 pub(super) fn run_if_enabled<A: Actionlike>(toggle_actions: Res<ToggleActions<A>>) -> ShouldRun {
     if toggle_actions.enabled {
diff --git a/tests/integration.rs b/tests/integration.rs
index 2e834411..0d704931 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -33,6 +33,12 @@ fn respect_fades(mut respect: ResMut<Respect>) {
     respect.0 = false;
 }
 
+fn remove_input_map(mut commands: Commands, query: Query<Entity, With<InputMap<Action>>>) {
+    for entity in query.iter() {
+        commands.entity(entity).remove::<InputMap<Action>>();
+    }
+}
+
 #[derive(Component)]
 struct Player;
 
@@ -162,6 +168,47 @@ fn disable_input() {
     assert_eq!(*respect, Respect(false));
 }
 
+#[test]
+fn release_when_input_map_removed() {
+    use bevy::input::InputPlugin;
+
+    let mut app = App::new();
+
+    // Spawn a player and create a global action state.
+    app.add_plugins(MinimalPlugins)
+        .add_plugin(InputPlugin)
+        .add_plugin(InputManagerPlugin::<Action>::default())
+        .add_startup_system(spawn_player)
+        .init_resource::<ActionState<Action>>()
+        .insert_resource(InputMap::<Action>::new([(KeyCode::F, Action::PayRespects)]))
+        .init_resource::<Respect>()
+        .add_system(pay_respects)
+        .add_system(remove_input_map)
+        .add_system_to_stage(CoreStage::PreUpdate, respect_fades);
+
+    // Press F to pay respects
+    app.send_input(KeyCode::F);
+    app.update();
+    let respect = app.world.resource::<Respect>();
+    assert_eq!(*respect, Respect(true));
+
+    // Remove the InputMap
+    app.world.remove_resource::<InputMap<Action>>();
+    // Needs an extra frame for the resource removed detection to release inputs
+    app.update();
+
+    // Now, all respect has faded
+    app.update();
+    let respect = app.world.resource::<Respect>();
+    assert_eq!(*respect, Respect(false));
+
+    // And even pressing F cannot bring it back
+    app.send_input(KeyCode::F);
+    app.update();
+    let respect = app.world.resource::<Respect>();
+    assert_eq!(*respect, Respect(false));
+}
+
 #[test]
 #[cfg(feature = "ui")]
 fn action_state_driver() {