Cloning the Repo onto the VM and clean-up

File Upload Service/streamlitdw/File upload service files/data-lakehouse/app/streamlitdw_fe.py → File Upload Service/app/backup/streamlitdw_fe_backup.py

@@ -1,60 +1,60 @@
-import streamlit as st
-from minio import Minio
-from minio.error import S3Error
-from dotenv import load_dotenv
-import io
-import os
-
-
-# Load environment variables
-load_dotenv()
-
-# Check the environment variables 
-access_key = os.getenv('AWS_ACCESS_KEY_ID')
-secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
-
-# st.write("Access Key ID:", access_key)
-# st.write("Secret Access Key:", secret_key)
-
-# Set up MinIO client using the loaded environment variables
-minio_client = Minio(
-    "10.137.0.149:9000",  # MinIO server address
-    access_key=access_key,  
-    secret_key=secret_key,  
-    secure=False  
-)
-
-bucket_name = "file-upload-service-sl"
-
-def upload_to_minio(file, filename):
-    try:
-        # Convert the uploaded file to bytes
-        data = file.read()
-        file_stream = io.BytesIO(data)
-
-        # Upload file to file upload service
-        minio_client.put_object(
-            bucket_name, filename, file_stream, len(data)
-        )
-        st.success(f"File {filename} uploaded successfully to Data Warehouse.") #try and except block to capture upload issues 
-    except S3Error as e:
-        st.error(f"Failed to upload {filename} to DataWarehouse: {e}")
-
-def main():
-    st.title("File Upload to Redback Data Warehouse Server")
-
-    # File uploader
-    uploaded_file = st.file_uploader("Choose a file", type=["csv", "txt", "xlsx","json"])
-
-    if uploaded_file is not None:
-        # Display file details
-        st.write(f"**Filename:** {uploaded_file.name}")
-        st.write(f"**File type:** {uploaded_file.type}")
-        st.write(f"**File size:** {uploaded_file.size / (1024 * 1024):.2f} MB")
-
-        # save file option
-        if st.button("Upload to Data Warehouse"):
-            upload_to_minio(uploaded_file, uploaded_file.name)
-
-if __name__ == "__main__":
+import streamlit as st
+from minio import Minio
+from minio.error import S3Error
+from dotenv import load_dotenv
+import io
+import os
+
+
+# Load environment variables
+load_dotenv()
+
+# Check the environment variables 
+access_key = os.getenv('AWS_ACCESS_KEY_ID')
+secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+
+# st.write("Access Key ID:", access_key)
+# st.write("Secret Access Key:", secret_key)
+
+# Set up MinIO client using the loaded environment variables
+minio_client = Minio(
+    "10.137.0.149:9000",  # MinIO server address
+    access_key=access_key,  
+    secret_key=secret_key,  
+    secure=False  
+)
+
+bucket_name = "file-upload-service-sl"
+
+def upload_to_minio(file, filename):
+    try:
+        # Convert the uploaded file to bytes
+        data = file.read()
+        file_stream = io.BytesIO(data)
+
+        # Upload file to file upload service
+        minio_client.put_object(
+            bucket_name, filename, file_stream, len(data)
+        )
+        st.success(f"File {filename} uploaded successfully to Data Warehouse.") #try and except block to capture upload issues 
+    except S3Error as e:
+        st.error(f"Failed to upload {filename} to DataWarehouse: {e}")
+
+def main():
+    st.title("File Upload to Redback Data Warehouse Server")
+
+    # File uploader
+    uploaded_file = st.file_uploader("Choose a file", type=["csv", "txt", "xlsx","json"])
+
+    if uploaded_file is not None:
+        # Display file details
+        st.write(f"**Filename:** {uploaded_file.name}")
+        st.write(f"**File type:** {uploaded_file.type}")
+        st.write(f"**File size:** {uploaded_file.size / (1024 * 1024):.2f} MB")
+
+        # save file option
+        if st.button("Upload to Data Warehouse"):
+            upload_to_minio(uploaded_file, uploaded_file.name)
+
+if __name__ == "__main__":
     main()

File Upload Service/app/streamlitdw_fe.py

@@ -0,0 +1,78 @@
+import streamlit as st
+from minio import Minio
+from minio.error import S3Error
+from dotenv import load_dotenv
+import io
+import os
+import datetime
+
+# Load environment variables
+load_dotenv()
+
+# Check the environment variables 
+access_key = os.getenv('AWS_ACCESS_KEY_ID')
+secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+
+# Set up MinIO client using the loaded environment variables
+minio_client = Minio(
+    "10.137.0.149:9000",  # MinIO server address
+    access_key=access_key,  
+    secret_key=secret_key,  
+    secure=False  
+)
+
+bucket_name = "file-upload-service-sl"
+
+def validate_filename(name):
+    # Ensure the name is alphanumeric (you can expand this with more rules if needed)
+    return name.isalnum()
+
+def generate_custom_filename(project, base_name, original_filename):
+    # Extract file extension
+    file_extension = original_filename.split(".")[-1]
+    # Generate a custom name with the project prefix, base name, and a date (YYYYMMDD)
+    date_stamp = datetime.datetime.now().strftime("%Y%m%d")
+    custom_filename = f"{project}/{base_name}_{date_stamp}.{file_extension}"  # Use project as folder prefix
+    return custom_filename
+
+def upload_to_minio(file, filename):
+    try:
+        # Convert the uploaded file to bytes
+        data = file.read()
+        file_stream = io.BytesIO(data)
+
+        # Upload file to MinIO, using filename with the project prefix as the object name
+        minio_client.put_object(
+            bucket_name, filename, file_stream, len(data)
+        )
+        st.success(f"File {filename} uploaded successfully to Data Warehouse.")
+    except S3Error as e:
+        st.error(f"Failed to upload {filename} to Data Warehouse: {e}")
+
+def main():
+    st.title("File Upload to Redback Data Warehouse Server")
+
+    # Project selection dropdown
+    project = st.selectbox("Select Project", options=["project1", "project2", "project3", "project4", "project5", "other"])
+
+    # File uploader
+    uploaded_file = st.file_uploader("Choose a file", type=["csv", "txt", "xlsx", "json"])
+
+    if uploaded_file is not None:
+        base_name = st.text_input("Enter base name for the file:")
+
+        if base_name and validate_filename(base_name):
+            # Generate the custom filename with the project prefix
+            custom_filename = generate_custom_filename(project, base_name, uploaded_file.name)
+            # Display file details
+            st.write(f"**Filename:** {custom_filename}")
+            st.write(f"**File type:** {uploaded_file.type}")
+            st.write(f"**File size:** {uploaded_file.size / (1024 * 1024):.2f} MB")
+
+            if st.button("Upload to Data Warehouse"):
+                upload_to_minio(uploaded_file, custom_filename)
+        else:
+            st.warning("Please enter a valid base name. Only alphanumeric characters are allowed.")
+
+if __name__ == "__main__":
+    main()

File Upload Service/streamlitdw/File upload service files/data-lakehouse/flask/flaskapi_dw.py → File Upload Service/flask/flaskapi_dw.py

@@ -1,48 +1,48 @@
-from flask import Flask, jsonify, send_file, Response
-from minio import Minio
-from minio.error import S3Error
-from dotenv import load_dotenv
-import os
-import io
-
-app = Flask(__name__)
-
-# minio details
-MINIO_URL = "10.137.0.149:9000"
-ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID') 
-SECRET_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') 
-BUCKET_NAME = "file-upload-service-sl"
-
-# Initialise Mini
-minio_client = Minio(
-    MINIO_URL,
-    access_key=ACCESS_KEY,
-    secret_key=SECRET_KEY,
-    secure=False
-)
-
-# Endpoint to list files in the bucket
-@app.route('/list-files', methods=['GET'])
-def list_files():
-    try:
-        objects = minio_client.list_objects(BUCKET_NAME)
-        files = [obj.object_name for obj in objects]
-        return jsonify(files)
-    except S3Error as err:
-        return jsonify({"error": str(err)}), 500
-
-# Endpoint to download a file from the bucket
-@app.route('/download-file/<filename>', methods=['GET'])
-def download_file(filename):
-    try:
-        data = minio_client.get_object(BUCKET_NAME, filename)
-        return send_file(
-            io.BytesIO(data.read()),
-            attachment_filename=filename,
-            as_attachment=True
-        )
-    except S3Error as err:
-        return jsonify({"error": str(err)}), 500
-
-if __name__ == '__main__':
+from flask import Flask, jsonify, send_file, Response
+from minio import Minio
+from minio.error import S3Error
+from dotenv import load_dotenv
+import os
+import io
+
+app = Flask(__name__)
+
+# minio details
+MINIO_URL = "10.137.0.149:9000"
+ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID') 
+SECRET_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') 
+BUCKET_NAME = "file-upload-service-sl"
+
+# Initialise Mini
+minio_client = Minio(
+    MINIO_URL,
+    access_key=ACCESS_KEY,
+    secret_key=SECRET_KEY,
+    secure=False
+)
+
+# Endpoint to list files in the bucket
+@app.route('/list-files', methods=['GET'])
+def list_files():
+    try:
+        objects = minio_client.list_objects(BUCKET_NAME)
+        files = [obj.object_name for obj in objects]
+        return jsonify(files)
+    except S3Error as err:
+        return jsonify({"error": str(err)}), 500
+
+# Endpoint to download a file from the bucket
+@app.route('/download-file/<filename>', methods=['GET'])
+def download_file(filename):
+    try:
+        data = minio_client.get_object(BUCKET_NAME, filename)
+        return send_file(
+            io.BytesIO(data.read()),
+            attachment_filename=filename,
+            as_attachment=True
+        )
+    except S3Error as err:
+        return jsonify({"error": str(err)}), 500
+
+if __name__ == '__main__':
     app.run(host='0.0.0.0', port=5000) # runnning on 5000

gitignore.txt

@@ -0,0 +1 @@
+.env