From e862aa0ef8f99f3e5ffddbc96f203515f47508c3 Mon Sep 17 00:00:00 2001
From: Duncan Blythe <duncan@superduperdb.com>
Date: Mon, 11 Nov 2024 12:46:07 +0100
Subject: [PATCH] Release all temlates

---
 .github/workflows/ci_templates.yaml       |  2 +-
 CHANGELOG.md                              |  2 +-
 superduper/templates/__init__.py          | 33 ++++++++++++++++++-----
 templates/llm_finetuning/VERSION          |  2 +-
 templates/multimodal_image_search/VERSION |  2 +-
 templates/multimodal_video_search/VERSION |  2 +-
 templates/pdf_rag/VERSION                 |  2 +-
 templates/rag/VERSION                     |  2 +-
 templates/transfer_learning/VERSION       |  2 +-
 9 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci_templates.yaml b/.github/workflows/ci_templates.yaml
index 72ebb786a..80f61b8f6 100644
--- a/.github/workflows/ci_templates.yaml
+++ b/.github/workflows/ci_templates.yaml
@@ -52,7 +52,7 @@ jobs:
           templates=("${!template_set[@]}")
 
           # Generate a single-line JSON
-          matrix_json=$(printf '%s' "${templates[@]}" | jq -R -s 'if . == "" then {template: []} else {template: (./"\n" | map(select(. != "")))} end' | jq -c .)
+          matrix_json=$(printf '%s\n' "${templates[@]}" | jq -R -s 'if . == "" then {template: []} else {template: (./"\n" | map(select(. != "")))} end' | jq -c .)
 
           echo "matrix_json=$matrix_json"
           echo "Changed templates: ${templates[*]}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3aa69d126..e9e789c07 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 **Before you create a Pull Request, remember to update the Changelog with your changes.**
 
-## Changes Since Last Release
+## Changes Since Last Release 
 
 #### Changed defaults / behaviours 
 
diff --git a/superduper/templates/__init__.py b/superduper/templates/__init__.py
index 7d6f03b6b..9b9e624e2 100644
--- a/superduper/templates/__init__.py
+++ b/superduper/templates/__init__.py
@@ -5,19 +5,38 @@
 
 PARENT = Path(__file__).resolve().parent
 
+BASE_URL = 'https://superduper-public-templates.s3.us-east-2.amazonaws.com'
+
+VERSIONS = {
+    'llm_finetuning': '0.4.0',
+    'multimodal_image_search': '0.4.0',
+    'multimodal_video_search': '0.4.0',
+    'pdf_rag': '0.4.0',
+    'rag': '0.4.0',
+    'simple_rag': '0.4.0',
+    'transfer_learning': '0.4.0',
+}
+
+
+TEMPLATES = {k: BASE_URL + f'/{k}-{VERSIONS[k]}.zip' for k in VERSIONS}
+
 
 def ls():
     """List all available templates."""
-    return [
-        x.split('.')[0]
-        for x in os.listdir(PARENT)
-        if not x.startswith('.') and not x.startswith('__')
-    ]
+    return TEMPLATES
 
 
 def __getattr__(name: str):
-    t = Template.read(str(PARENT / f"{name}"))
-    requirements_path = str(PARENT / f"{name}" / "requirements.txt")
+    assert name in TEMPLATES
+    file = TEMPLATES[name].split('/')[-1]
+    if not os.path.exists(f'/tmp/{file}'):
+        import subprocess
+
+        subprocess.run(['curl', '-O', '-k', TEMPLATES[name]])
+        subprocess.run(['mv', file, f'/tmp/{file}'])
+        subprocess.run(['unzip', f'/tmp/{file}', '-d', f'/tmp/{file[:-4]}'])
+    t = Template.read(f'/tmp/{file[:-4]}')
+    requirements_path = f'{file[:-4]}/requirements.txt'
     if os.path.exists(requirements_path):
         with open(requirements_path, 'r') as f:
             t.requirements = [x.strip() for x in f.read().split('\n') if x]
diff --git a/templates/llm_finetuning/VERSION b/templates/llm_finetuning/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/llm_finetuning/VERSION
+++ b/templates/llm_finetuning/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file
diff --git a/templates/multimodal_image_search/VERSION b/templates/multimodal_image_search/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/multimodal_image_search/VERSION
+++ b/templates/multimodal_image_search/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file
diff --git a/templates/multimodal_video_search/VERSION b/templates/multimodal_video_search/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/multimodal_video_search/VERSION
+++ b/templates/multimodal_video_search/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file
diff --git a/templates/pdf_rag/VERSION b/templates/pdf_rag/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/pdf_rag/VERSION
+++ b/templates/pdf_rag/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file
diff --git a/templates/rag/VERSION b/templates/rag/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/rag/VERSION
+++ b/templates/rag/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file
diff --git a/templates/transfer_learning/VERSION b/templates/transfer_learning/VERSION
index 26c207022..60a2d3e96 100644
--- a/templates/transfer_learning/VERSION
+++ b/templates/transfer_learning/VERSION
@@ -1 +1 @@
-0.5.0.dev
\ No newline at end of file
+0.4.0
\ No newline at end of file