Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dynamic Forms #18

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions backend/app/routers/template.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import os
import yaml
from fastapi import APIRouter, HTTPException
from typing import List, Dict
from ..utils.logger import get_logger

router = APIRouter()
logger = get_logger(__name__)

TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "config", "templates")

@router.get("/templates")
async def list_templates() -> Dict[str, List[Dict[str, str]]]:
"""
List all available templates.
"""
try:
if not os.path.exists(TEMPLATES_DIR):
os.makedirs(TEMPLATES_DIR)
logger.info(f"Created templates directory at {TEMPLATES_DIR}")
return {"templates": []}

templates = []
for filename in os.listdir(TEMPLATES_DIR):
if filename.endswith('.yaml'):
template_id = filename[:-5] # Remove .yaml extension
try:
with open(os.path.join(TEMPLATES_DIR, filename), 'r') as f:
template_data = yaml.safe_load(f)
templates.append({
"id": template_id,
"name": template_data.get("name", template_id),
"description": template_data.get("description", "")
})
except Exception as e:
logger.error(f"Error loading template {filename}: {str(e)}")
continue

logger.info(f"Found {len(templates)} templates")
return {"templates": templates}
except Exception as e:
logger.error(f"Error loading templates: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error loading templates: {str(e)}")

@router.get("/templates/{template_id}")
async def get_template(template_id: str):
"""
Retrieve the template configuration for a specific template.
"""
template_path = os.path.join(TEMPLATES_DIR, f"{template_id}.yaml")

if not os.path.exists(template_path):
raise HTTPException(status_code=404, detail=f"Template {template_id} not found")

try:
with open(template_path, 'r') as f:
template = yaml.safe_load(f)
return template
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error loading template: {str(e)}")
72 changes: 72 additions & 0 deletions backend/config/templates/mysql.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: "MySQL Database"
description: "Template for MySQL database connections"
tabs:
info:
label: "Information"
description: "Basic information about the data contract"
fields:
- name: "title"
label: "Title"
type: "text"
required: true
placeholder: "My MySQL Data Contract"
hint: "A descriptive name for this data contract"

- name: "version"
label: "Version"
type: "text"
required: true
pattern: "^\\d+\\.\\d+\\.\\d+$"
placeholder: "1.0.0"
hint: "Semantic version (e.g. 1.0.0)"

- name: "description"
label: "Description"
type: "textarea"
required: true
rows: 4
hint: "Detailed description of this data contract"

server:
label: "Server"
description: "Database connection details"
fields:
- name: "host"
label: "Host"
type: "text"
required: true
placeholder: "localhost"
hint: "Database server hostname or IP address"

- name: "port"
label: "Port"
type: "number"
required: true
default: 3306
min: 1
max: 65535
hint: "Database server port (default: 3306)"

- name: "database"
label: "Database Name"
type: "text"
required: true
hint: "Name of the database to connect to"

- name: "username"
label: "Username"
type: "text"
required: true
hint: "Database user account"

- name: "password"
label: "Password"
type: "password"
required: true
hint: "Database user password"

- name: "ssl_enabled"
label: "Enable SSL"
type: "boolean"
default: false
hint: "Use SSL/TLS for connection"
116 changes: 116 additions & 0 deletions backend/config/templates/oracle.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
name: "Oracle Database Source"
description: "Template for Oracle database connections and table extractions"
tabs:
info:
label: "Information"
description: "Basic information about the data contract"
fields:
- name: "title"
label: "Title"
type: "text"
required: true
placeholder: "My Oracle Data Contract"
hint: "A descriptive name for this data contract"

- name: "version"
label: "Version"
type: "text"
required: true
pattern: "^\\d+\\.\\d+\\.\\d+$"
placeholder: "1.0.0"
hint: "Semantic version (e.g. 1.0.0)"

- name: "description"
label: "Description"
type: "textarea"
required: true
rows: 4
hint: "Detailed description of this data contract"

- name: "owner"
label: "Owner"
type: "text"
required: true
hint: "Team or person responsible for this data contract"

source:
label: "Source"
description: "Oracle database connection details"
fields:
- name: "host"
label: "Host"
type: "text"
required: true
placeholder: "db.example.com"
hint: "Database server hostname or IP address"

- name: "port"
label: "Port"
type: "number"
required: true
default: 1521
min: 1
max: 65535
hint: "Database server port (default: 1521)"

- name: "service_name"
label: "Service Name"
type: "text"
required: true
hint: "Oracle service name"

- name: "username"
label: "Username"
type: "text"
required: true
hint: "Database user account"

- name: "password_secret_id"
label: "Password Secret ID"
type: "text"
required: true
hint: "Secret ID for database password in secret manager"

schema:
label: "Schema"
description: "Configuration for data extraction"
fields:
- name: "schedule"
label: "Schedule"
type: "text"
required: true
placeholder: "0 4 * * *"
hint: "Cron expression for scheduling extractions"

- name: "table"
label: "Source Table"
type: "text"
required: true
placeholder: "SCHEMA.TABLE_NAME"
hint: "Fully qualified table name to extract from"

- name: "bucket_name"
label: "Bucket Name"
type: "text"
required: true
hint: "GCS bucket name for storing extracted data"

compute:
label: "Compute"
description: "Compute resources for extraction"
fields:
- name: "memory_limit"
label: "Memory Limit (GB)"
type: "number"
default: 4
min: 1
max: 16
hint: "Memory limit in gigabytes"

- name: "cpu_limit"
label: "CPU Limit"
type: "number"
default: 1
min: 0.1
max: 4
hint: "CPU cores limit"
137 changes: 137 additions & 0 deletions backend/config/templates/sftp.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
name: "SFTP CSV Source"
description: "Template for SFTP CSV file extractions"
tabs:
info:
label: "Information"
description: "Basic information about the data contract"
fields:
- name: "title"
label: "Title"
type: "text"
required: true
placeholder: "My SFTP Data Contract"
hint: "A descriptive name for this data contract"

- name: "version"
label: "Version"
type: "text"
required: true
pattern: "^\\d+\\.\\d+\\.\\d+$"
placeholder: "1.0.0"
hint: "Semantic version (e.g. 1.0.0)"

- name: "description"
label: "Description"
type: "textarea"
required: true
rows: 4
hint: "Detailed description of this data contract"

- name: "owner"
label: "Owner"
type: "text"
required: true
hint: "Team or person responsible for this data contract"

source:
label: "Source"
description: "SFTP server connection details"
fields:
- name: "host"
label: "Host"
type: "text"
required: true
placeholder: "sftp.example.com"
hint: "SFTP server hostname or IP address"

- name: "port"
label: "Port"
type: "number"
required: true
default: 22
min: 1
max: 65535
hint: "SFTP server port (default: 22)"

- name: "username"
label: "Username"
type: "text"
required: true
hint: "SFTP user account"

- name: "password_secret_id"
label: "Password Secret ID"
type: "text"
required: true
hint: "Secret ID for SFTP password in secret manager"

schema:
label: "Schema"
description: "CSV file configuration"
fields:
- name: "file_name"
label: "Remote File Path"
type: "text"
required: true
placeholder: "/path/to/file.csv"
hint: "Full path to CSV file on SFTP server"

- name: "delimiter"
label: "Delimiter"
type: "text"
required: true
default: "|"
maxLength: 1
hint: "CSV field delimiter character"

- name: "encoding"
label: "File Encoding"
type: "select"
required: true
default: "UTF-8"
options:
- "UTF-8"
- "ISO-8859-1"
- "ASCII"
hint: "Character encoding of the CSV file"

- name: "date_format"
label: "Date Format"
type: "text"
required: true
default: "dd-MMM-yyyy"
hint: "Format for parsing date fields"

- name: "has_header"
label: "Has Header"
type: "boolean"
default: true
hint: "First row contains column names"

- name: "mode"
label: "Load Mode"
type: "select"
required: true
default: "FAILFAST"
options:
- "FAILFAST"
- "DROPMALFORMED"
- "PERMISSIVE"
hint: "How to handle malformed records"

compute:
label: "Compute"
description: "Schedule and storage configuration"
fields:
- name: "schedule"
label: "Schedule"
type: "text"
required: true
placeholder: "0 4 * * *"
hint: "Cron expression for scheduling extractions"

- name: "bucket_name"
label: "Bucket Name"
type: "text"
required: true
hint: "GCS bucket name for storing extracted data"
Loading
Loading