diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 56a71327..dda70b37 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -14,6 +14,14 @@ Change Log
 Unreleased
 ~~~~~~~~~~
 
+[0.2.4] - 2022-05-23
+~~~~~~~~~~~~~~~~~~~~
+
+Added
++++++++
+
+* Added a check to validate that pip.txt requirements are installed immediately after upgrading pip.txt in Makefile's upgrade target
+
 [0.2.3] - 2022-04-12
 ~~~~~~~~~~~~~~~~~~~~
 
diff --git a/repo_health/__init__.py b/repo_health/__init__.py
index 0ad046b1..1d9132f2 100644
--- a/repo_health/__init__.py
+++ b/repo_health/__init__.py
@@ -8,7 +8,7 @@
 import pytest
 import dockerfile
 
-__version__ = "0.2.3"
+__version__ = "0.2.4"
 
 
 GITHUB_URL_PATTERN = r"github.com[/:](?P<org_name>[^/]+)/(?P<repo_name>[^/]+).*#egg=(?P<package>[^\/]+).*"
diff --git a/repo_health/check_makefile.py b/repo_health/check_makefile.py
index ebcf87f0..ce4f9ffd 100644
--- a/repo_health/check_makefile.py
+++ b/repo_health/check_makefile.py
@@ -42,3 +42,29 @@ def check_has_make_target(makefile, all_results):
         match = re.search(regex_pattern, makefile, re.MULTILINE)
         if match:
             all_results[module_dict_key][target] = True
+
+@health_metadata(
+    [module_dict_key],
+    {
+        "pip-installed": "check if pip.txt was installed immediately after upgrade"
+    }
+)
+def check_upgrade_script(makefile, all_results):
+    """
+    Checks if pip installed after upgrading pip.txt
+    """
+    upgrade_targets = re.finditer("^upgrade:", makefile, re.MULTILINE)
+
+    for i in upgrade_targets:
+        content = makefile[i.end():]
+        upgrade_script = content[:re.search("^[a-zA-Z_]+: ", content, re.MULTILINE).start()]
+        update_commands = (r"(\n\t(\$\(PIP_COMPILE\)|pip-compile)(.*?)((requirements/pip\.txt requirements/pip\.in)"
+                           r"|(requirements/pip-tools\.txt requirements/pip-tools\.in))){2}")
+        install_commands = r"(\n\t(pip install)(.*?)(requirements/pip.txt|requirements/pip-tools.txt)){2}"
+        regex_pattern = "".join([update_commands, install_commands])
+        match = re.search(regex_pattern, upgrade_script, re.MULTILINE)
+        if match:
+            all_results[module_dict_key]["pip-installed"] = True
+            return
+
+    all_results[module_dict_key]["pip-installed"] = False
diff --git a/tests/fake_repos/makefile_repo2/Makefile b/tests/fake_repos/makefile_repo2/Makefile
index 0233b1ee..9ce1ec5e 100644
--- a/tests/fake_repos/makefile_repo2/Makefile
+++ b/tests/fake_repos/makefile_repo2/Makefile
@@ -18,7 +18,10 @@ piptools:
 
 export CUSTOM_COMPILE_COMMAND = make upgrade
 upgrade: piptools ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip-compile --rebuild --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in
+	pip-compile --rebuild --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in
+	pip install -qr requirements/pip.txt
+	pip install -qr requirements/pip-tools.txt
 	pip-compile --rebuild --upgrade -o requirements/base.txt requirements/base.in
 	pip-compile --rebuild --upgrade -o requirements/test.txt requirements/test.in
 	pip-compile --rebuild --upgrade -o requirements/docs.txt requirements/docs.in
diff --git a/tests/test_check_makefile.py b/tests/test_check_makefile.py
index a25bf389..1a8617f8 100644
--- a/tests/test_check_makefile.py
+++ b/tests/test_check_makefile.py
@@ -4,6 +4,7 @@
 from repo_health.check_makefile import (
     module_dict_key,
     check_has_make_target,
+    check_upgrade_script,
     output_keys,
 )
 
@@ -41,3 +42,18 @@ def test_check_file_existence(fake_repo, flag_list):
 
     for key, desc in output_keys.items():
         assert all_results[module_dict_key][key] == flag_list[key]
+
+
+@pytest.mark.parametrize("fake_repo, flag", [
+    ("makefile_repo1",
+    {"pip-installed": False
+    }),
+    ("makefile_repo2",
+    {"pip-installed": True
+    })])
+def test_check_upgrade_script(fake_repo, flag):
+    repo_path = get_repo_path('fake_repos/' + fake_repo)
+    all_results = {module_dict_key: {}}
+    file = open(repo_path + '/Makefile', 'r')
+    check_upgrade_script(file.read(), all_results)
+    assert all_results[module_dict_key]["pip-installed"] == flag["pip-installed"]