From 4470b670fb967caf6e3300f7ea861bab1dd765f1 Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 13:45:17 -0400
Subject: [PATCH 1/9] new: fix case-sensitivity for snyk updates

- Attempt to check if package was previously being specified before automatically adding it because  of the scan
---
 tools/snyk-update.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tools/snyk-update.py b/tools/snyk-update.py
index 893fe603..e9852286 100644
--- a/tools/snyk-update.py
+++ b/tools/snyk-update.py
@@ -7,6 +7,8 @@
 scan = json.load(scan_file)
 scan_file.close()
 
+update = False
+
 # Make changes to fixable results
 remediations = scan['remediation']['pin']
 for k, v in remediations.items():
@@ -14,11 +16,16 @@
     new_version = v['upgradeTo'].split('@')[1]
     print(package, old_version, new_version)
 
-    # Remove old version
-    os.system('sed -i "/%s/d" ckan/requirements.in' % (package + "==" + old_version))
-    os.system('sed -i "/%s/d" ckan/requirements.in' % (package + ">=" + old_version))
+    # Check to see if package was explicitly included
+    with open('ckan/requirements.in', 'r') as source:
+        all_requirements = source.readlines()
+    if package in ', '.join(all_requirements):
+        update = True
 
-    # Add new version
-    os.system("echo '%s' >> ckan/requirements.in" % (package + ">=" + new_version))
+    if update:
+        # Remove old version
+        os.system('sed -i "/^%s\\(=\\|>\\|$\\)/Id" ckan/requirements.in' % (package))
+        # Add new version if it was already being specified
+        os.system("echo '%s' >> ckan/requirements.in" % (package + ">=" + new_version))
 
 # TODO: Handle unfixable results

From 3a4833774c5d865d3efc5742b10df1a26d181a9f Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 14:35:34 -0400
Subject: [PATCH 2/9] fix: reset update to False for the next package

Otherwise, the first package will set update for every package
---
 tools/snyk-update.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/snyk-update.py b/tools/snyk-update.py
index e9852286..a5d443de 100644
--- a/tools/snyk-update.py
+++ b/tools/snyk-update.py
@@ -7,25 +7,25 @@
 scan = json.load(scan_file)
 scan_file.close()
 
-update = False
-
 # Make changes to fixable results
 remediations = scan['remediation']['pin']
 for k, v in remediations.items():
+    update = False
     package, old_version = k.split('@')
     new_version = v['upgradeTo'].split('@')[1]
     print(package, old_version, new_version)
 
     # Check to see if package was explicitly included
+    # If it was not, then we only care about updating
+    # requirements.txt, not requirements.in
+    # 'make update-dependencies' will update requirements.txt
     with open('ckan/requirements.in', 'r') as source:
         all_requirements = source.readlines()
     if package in ', '.join(all_requirements):
         update = True
 
     if update:
-        # Remove old version
         os.system('sed -i "/^%s\\(=\\|>\\|$\\)/Id" ckan/requirements.in' % (package))
-        # Add new version if it was already being specified
         os.system("echo '%s' >> ckan/requirements.in" % (package + ">=" + new_version))
 
 # TODO: Handle unfixable results

From d9936b700bc210ebab9cb9220c646383e6ffa498 Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 14:36:15 -0400
Subject: [PATCH 3/9] fix: ckanext-qa doesn't like newer setuptools in
 virtualenv?

---
 ckan/freeze-requirements.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ckan/freeze-requirements.sh b/ckan/freeze-requirements.sh
index 016c5c02..d5adcf98 100755
--- a/ckan/freeze-requirements.sh
+++ b/ckan/freeze-requirements.sh
@@ -14,6 +14,7 @@ trap cleanup EXIT
 pip3 install virtualenv
 
 virtualenv $venv
+${venv}/bin/pip3 install setuptools==67.1.0
 ${venv}/bin/pip3 install -r /app/ckan/requirements.in
 
-${venv}/bin/pip3 freeze --all > /app/ckan/requirements.txt
\ No newline at end of file
+${venv}/bin/pip3 freeze --all > /app/ckan/requirements.txt

From 373f241a566732a8001b1ff8d83431de4c4fc76f Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 14:37:02 -0400
Subject: [PATCH 4/9] new: add special docker-compose service for requirements

This should help cut down on build+install time with the snyk scan
---
 Makefile           | 2 +-
 docker-compose.yml | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cbdfe96d..f7ed97d3 100644
--- a/Makefile
+++ b/Makefile
@@ -36,7 +36,7 @@ up:
 	docker-compose up $(ARGS)
 
 update-dependencies:
-	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+	docker-compose run --rm -T ckan-requirements /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
 
 # ###############################################
 # Test commands
diff --git a/docker-compose.yml b/docker-compose.yml
index ca27d135..e3724233 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -52,6 +52,14 @@ services:
       - ./tests/harvest-sources:/usr/share/nginx/html
       - ./tests/nginx.conf:/etc/nginx/conf.d/default.conf
 
+  ckan-requirements:
+    image: ghcr.io/gsa/catalog.data.gov:latest
+    build:
+      context: ckan/
+    env_file: .env
+    volumes:
+      - .:/app
+
 volumes:
   ckan_storage:
   pg_data:

From fa213ecdbc097fcdda6b01d60b434e1538c77c86 Mon Sep 17 00:00:00 2001
From: nickumia-reisys <datagovhelp@gsa.gov>
Date: Mon, 3 Apr 2023 18:44:33 +0000
Subject: [PATCH 5/9] Update Pip Requirements

---
 ckan/requirements.in  |  1 +
 ckan/requirements.txt | 34 +++++++++++++++++-----------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/ckan/requirements.in b/ckan/requirements.in
index 2435ebb2..f84cb244 100644
--- a/ckan/requirements.in
+++ b/ckan/requirements.in
@@ -116,3 +116,4 @@ gevent>=21.8.0
 newrelic
 certifi>=2022.12.7
 cryptography>=39.0.1
+redis>=4.5.4
diff --git a/ckan/requirements.txt b/ckan/requirements.txt
index 06f5f201..555026b3 100644
--- a/ckan/requirements.txt
+++ b/ckan/requirements.txt
@@ -3,32 +3,32 @@ async-timeout==4.0.2
 Babel==2.9.1
 Beaker==1.11.0
 bleach==3.3.0
-boto3==1.26.73
-botocore==1.29.73
+boto3==1.26.104
+botocore==1.29.104
 certifi==2022.12.7
 cffi==1.15.1
 chardet==3.0.4
 ckan==2.9.8
--e git+https://github.com/ckan/ckanext-archiver.git@c96e3c81bfc430cdb0372f3307c7abd4109a80f1#egg=ckanext_archiver
+-e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver
 ckanext-datagovcatalog==0.0.5
 ckanext-datagovtheme==0.1.24
 ckanext-datajson==0.1.17
 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@618928be5a211babafc45103a72b6aab4642e964
-ckanext-envvars==0.0.2
+ckanext-envvars==0.0.3
 ckanext-geodatagov==0.1.33
 ckanext-googleanalyticsbasic==0.2.0
--e git+https://github.com/ckan/ckanext-harvest.git@89a98d7ff5aa3445d8158921669b8d0b04fa41c3#egg=ckanext_harvest
+-e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest
 ckanext-metrics-dashboard==0.1.5
 -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa
 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report
 ckanext-saml2auth @ git+https://github.com/keitaroinc/ckanext-saml2auth.git@b2c6cfc2f9d6179601017f2d6530cf4ee5331f00
--e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial
+-e git+https://github.com/ckan/ckanext-spatial.git@944ae4dd973f37a291514823b27607ea296af05a#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==7.1.2
-cryptography==39.0.1
+cryptography==40.0.1
 defusedxml==0.7.1
 dominate==2.4.0
-elementpath==4.0.1
+elementpath==4.1.0
 fanstatic==1.1
 feedgen==0.9.0
 Flask==2.0.0
@@ -42,7 +42,7 @@ greenlet==2.0.2
 gunicorn==20.1.0
 html5lib==1.1
 idna==2.10
-importlib-resources==5.10.2
+importlib-resources==5.12.0
 isodate==0.6.1
 itsdangerous==2.1.2
 Jinja2==3.0.0
@@ -54,7 +54,7 @@ Mako==1.2.4
 Markdown==3.1.1
 MarkupSafe==2.1.2
 messytables==0.15.2
-newrelic==8.7.0
+newrelic==8.8.0
 nose==1.3.7
 numpy==1.24.2
 OWSLib==0.18.0
@@ -63,7 +63,7 @@ passlib==1.7.3
 PasteDeploy==2.0.1
 pathtools==0.1.2
 pika==1.3.1
-pip==23.0
+pip==23.0.1
 ply==3.11
 polib==1.0.7
 progressbar==2.5
@@ -71,7 +71,7 @@ progressbar2==3.53.3
 psycopg2==2.8.6
 pycparser==2.21
 PyJWT==2.4.0
-pyOpenSSL==23.0.0
+pyOpenSSL==23.1.1
 pyparsing==3.0.9
 pyproj==2.6.1
 pysaml2==7.0.1
@@ -85,7 +85,7 @@ PyUtilib==5.7.1
 PyYAML==5.4
 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219
 rdflib==4.2.2
-redis==4.5.1
+redis==4.5.4
 repoze.lru==0.7
 repoze.who==2.3
 requests==2.25.0
@@ -102,14 +102,14 @@ SQLAlchemy==1.3.5
 sqlparse==0.4.2
 tzlocal==1.3
 unicodecsv==0.14.1
-urllib3==1.26.14
+urllib3==1.26.15
 webassets==0.12.1
 webencodings==0.5.1
 WebOb==1.8.7
 Werkzeug==2.0.0
 wheel==0.38.4
 xlrd==2.0.1
-xmlschema==2.2.1
-zipp==3.13.0
+xmlschema==2.2.2
+zipp==3.15.0
 zope.event==4.6
-zope.interface==5.5.2
+zope.interface==6.0

From 1ee3095082d4faecd913fa0c11b24dd1fc42f02c Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 14:48:37 -0400
Subject: [PATCH 6/9] revert: standalone ckan-requirements is not any faster

---
 Makefile           | 2 +-
 docker-compose.yml | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index f7ed97d3..cbdfe96d 100644
--- a/Makefile
+++ b/Makefile
@@ -36,7 +36,7 @@ up:
 	docker-compose up $(ARGS)
 
 update-dependencies:
-	docker-compose run --rm -T ckan-requirements /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
 
 # ###############################################
 # Test commands
diff --git a/docker-compose.yml b/docker-compose.yml
index e3724233..ca27d135 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -52,14 +52,6 @@ services:
       - ./tests/harvest-sources:/usr/share/nginx/html
       - ./tests/nginx.conf:/etc/nginx/conf.d/default.conf
 
-  ckan-requirements:
-    image: ghcr.io/gsa/catalog.data.gov:latest
-    build:
-      context: ckan/
-    env_file: .env
-    volumes:
-      - .:/app
-
 volumes:
   ckan_storage:
   pg_data:

From 6297907cfc47784f4151cd00e581ff9b77ec8b4c Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <nicholas.kumia@reisystems.com>
Date: Mon, 3 Apr 2023 14:55:20 -0400
Subject: [PATCH 7/9] revert: only update if explicit reference did not work

---
 tools/snyk-update.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tools/snyk-update.py b/tools/snyk-update.py
index a5d443de..71ed6320 100644
--- a/tools/snyk-update.py
+++ b/tools/snyk-update.py
@@ -10,22 +10,15 @@
 # Make changes to fixable results
 remediations = scan['remediation']['pin']
 for k, v in remediations.items():
-    update = False
     package, old_version = k.split('@')
     new_version = v['upgradeTo'].split('@')[1]
     print(package, old_version, new_version)
 
-    # Check to see if package was explicitly included
-    # If it was not, then we only care about updating
-    # requirements.txt, not requirements.in
-    # 'make update-dependencies' will update requirements.txt
-    with open('ckan/requirements.in', 'r') as source:
-        all_requirements = source.readlines()
-    if package in ', '.join(all_requirements):
-        update = True
+    # TODO: Handle case when vulnerable package isn't explicitly in requirements.in
 
-    if update:
-        os.system('sed -i "/^%s\\(=\\|>\\|$\\)/Id" ckan/requirements.in' % (package))
-        os.system("echo '%s' >> ckan/requirements.in" % (package + ">=" + new_version))
+    # Remove old version
+    os.system('sed -i "/^%s\\(=\\|>\\|$\\)/Id" ckan/requirements.in' % (package))
+    # Add new version
+    os.system("echo '%s' >> ckan/requirements.in" % (package + ">=" + new_version))
 
 # TODO: Handle unfixable results

From f9642679366c5f15d729b2b6da0241460055a082 Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <85196563+nickumia-reisys@users.noreply.github.com>
Date: Mon, 3 Apr 2023 16:55:07 -0400
Subject: [PATCH 8/9] revert: spatial is not ready for upgrade

---
 ckan/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/requirements.txt b/ckan/requirements.txt
index 555026b3..aa5259e5 100644
--- a/ckan/requirements.txt
+++ b/ckan/requirements.txt
@@ -22,7 +22,7 @@ ckanext-metrics-dashboard==0.1.5
 -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa
 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report
 ckanext-saml2auth @ git+https://github.com/keitaroinc/ckanext-saml2auth.git@b2c6cfc2f9d6179601017f2d6530cf4ee5331f00
--e git+https://github.com/ckan/ckanext-spatial.git@944ae4dd973f37a291514823b27607ea296af05a#egg=ckanext_spatial
+-e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==7.1.2
 cryptography==40.0.1

From 6c2f5274014d49bc6ecbc386ac4ca88aa4f49f20 Mon Sep 17 00:00:00 2001
From: Nicholas Kumia <85196563+nickumia-reisys@users.noreply.github.com>
Date: Tue, 4 Apr 2023 10:31:45 -0400
Subject: [PATCH 9/9] revert: harvest? maybe something breaking here?

---
 ckan/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/requirements.txt b/ckan/requirements.txt
index aa5259e5..5bfcb490 100644
--- a/ckan/requirements.txt
+++ b/ckan/requirements.txt
@@ -17,7 +17,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@618928be5a211babafc45103
 ckanext-envvars==0.0.3
 ckanext-geodatagov==0.1.33
 ckanext-googleanalyticsbasic==0.2.0
--e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest
+-e git+https://github.com/ckan/ckanext-harvest.git@89a98d7ff5aa3445d8158921669b8d0b04fa41c3#egg=ckanext_harvest
 ckanext-metrics-dashboard==0.1.5
 -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa
 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report