Release all temlates

.github/workflows/ci_templates.yaml

@@ -52,7 +52,7 @@ jobs:
           templates=("${!template_set[@]}")
 
           # Generate a single-line JSON
-          matrix_json=$(printf '%s' "${templates[@]}" | jq -R -s 'if . == "" then {template: []} else {template: (./"\n" | map(select(. != "")))} end' | jq -c .)
+          matrix_json=$(printf '%s\n' "${templates[@]}" | jq -R -s 'if . == "" then {template: []} else {template: (./"\n" | map(select(. != "")))} end' | jq -c .)
 
           echo "matrix_json=$matrix_json"
           echo "Changed templates: ${templates[*]}"

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 **Before you create a Pull Request, remember to update the Changelog with your changes.**
 
-## Changes Since Last Release
+## Changes Since Last Release 
 
 #### Changed defaults / behaviours 
 

superduper/templates/__init__.py

@@ -5,19 +5,38 @@
 
 PARENT = Path(__file__).resolve().parent
 
+BASE_URL = 'https://superduper-public-templates.s3.us-east-2.amazonaws.com'
+
+VERSIONS = {
+    'llm_finetuning': '0.4.0',
+    'multimodal_image_search': '0.4.0',
+    'multimodal_video_search': '0.4.0',
+    'pdf_rag': '0.4.0',
+    'rag': '0.4.0',
+    'simple_rag': '0.4.0',
+    'transfer_learning': '0.4.0',
+}
+
+
+TEMPLATES = {k: BASE_URL + f'/{k}-{VERSIONS[k]}.zip' for k in VERSIONS}
+
 
 def ls():
     """List all available templates."""
-    return [
-        x.split('.')[0]
-        for x in os.listdir(PARENT)
-        if not x.startswith('.') and not x.startswith('__')
-    ]
+    return TEMPLATES
 
 
 def __getattr__(name: str):
-    t = Template.read(str(PARENT / f"{name}"))
-    requirements_path = str(PARENT / f"{name}" / "requirements.txt")
+    assert name in TEMPLATES
+    file = TEMPLATES[name].split('/')[-1]
+    if not os.path.exists(f'/tmp/{file}'):
+        import subprocess
+
+        subprocess.run(['curl', '-O', '-k', TEMPLATES[name]])
+        subprocess.run(['mv', file, f'/tmp/{file}'])
+        subprocess.run(['unzip', f'/tmp/{file}', '-d', f'/tmp/{file[:-4]}'])
+    t = Template.read(f'/tmp/{file[:-4]}')
+    requirements_path = f'{file[:-4]}/requirements.txt'
     if os.path.exists(requirements_path):
         with open(requirements_path, 'r') as f:
             t.requirements = [x.strip() for x in f.read().split('\n') if x]

templates/llm_finetuning/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0

templates/multimodal_image_search/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0

templates/multimodal_video_search/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0

templates/pdf_rag/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0

templates/rag/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0

templates/transfer_learning/VERSION

@@ -1 +1 @@
-0.5.0.dev
+0.4.0