[ feat ]: Crafting Pipeline and RPC for ML

.github/workflows/lstm_gru_pipeline.yaml

@@ -0,0 +1,393 @@
+name: LSTM GRU Pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'   
+  schedule:
+    - cron: "0 11 * * *"
+    # 18 - 7 = 11
+
+
+jobs:
+  scraping_extracting:
+    name: Scraping and Extracting
+    runs-on: ubuntu-latest
+    environment: Scraping and Extraction
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          lfs: true
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Scraping Yahoo Finance
+        run: |
+          mkdir datasets
+          go run scraper.go \
+            --symbols-file=./postman/symbols.json
+
+      - name: Zip Datasets
+        run: zip -r datasets.zip datasets
+
+      - name: Store Datasets to Google Drive
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
+          filename: datasets.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: datasets.zip
+          overwrite: "true"
+
+      - name: Upload Artifact (datasets)
+        uses: actions/upload-artifact@v3
+        with:
+          name: datasets
+          path: datasets.zip
+
+      - name: Remove Temporarary Files and Directories
+        run: |
+          rm datasets.zip
+          rm -rf datasets
+
+
+  preprocessing_training:
+    name: Preprocessing and Training 
+    runs-on: ubuntu-latest
+    needs: scraping_extracting
+    environment: Preprocessing and Training 
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          lfs: true
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Install Libraries
+        run: pip install -r requirements.txt
+
+      - name: Download Artifact (datasets)
+        uses: actions/download-artifact@v3
+        with:
+          name: datasets
+
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y \
+              gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
+      - name: Modeling and Training
+        run: |
+          unzip datasets.zip
+
+          mkdir models
+          mkdir pickles
+          mkdir posttrained
+
+          python trainingcli.py \
+            --epochs=200        \
+            --batchs=32         \
+            --sequences=60      \
+            --algorithm=LSTM_GRU
+          
+      - name: Zip Posttrained, Models, and Pickles
+        run: |
+          zip -r models.zip models
+          zip -r pickles.zip pickles
+          zip -r posttrained.zip posttrained
+          
+      - name: Store Models to Google Drive
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
+          filename: models.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: models.zip
+          overwrite: "true"
+
+      - name: Store Pickles to Google Drive
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
+          filename: pickles.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: pickles.zip
+          overwrite: "true"
+
+      - name: Store Posttrained to Google Drive
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
+          filename: posttrained.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: posttrained.zip
+          overwrite: "true"
+
+      - name: Remove Temporarary Files and Directories
+        run: |
+          rm models.zip
+          rm pickles.zip
+          rm datasets.zip
+          rm posttrained.zip
+
+          rm -rf models
+          rm -rf pickles
+          rm -rf datasets
+          rm -rf posttrained
+
+
+  tebakaja_stock_space-0:
+    name: stock-forecast-svc-0
+    runs-on: ubuntu-latest
+    needs: preprocessing_training
+    environment: Deployment
+
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      SPACE_NAME: tebakaja_stock_space-0
+      HF_USERNAME: tebakaja
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1000
+
+      - name: Check git status
+        run: git status
+
+      - name: Configure git
+        run: |
+          git config --local user.email "alfariqyraihan@gmail.com"
+          git config --local user.name "qywok"
+
+      - name: Pull changes from remote
+        run: |
+          git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
+          (git merge --strategy-option theirs)
+
+      - name: Add and commit changes
+        run: |
+          git add -A 
+          git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+      - name: Push to Hugging Face
+        run: |
+          git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
+
+
+  tebakaja_stock_space-1:
+    name: stock-forecast-svc-1
+    runs-on: ubuntu-latest
+    needs: preprocessing_training
+    environment: Deployment
+
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      SPACE_NAME: tebakaja_stock_space-1
+      HF_USERNAME: tebakaja
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1000
+
+      - name: Check git status
+        run: git status
+
+      - name: Configure git
+        run: |
+          git config --local user.email "alfariqyraihan@gmail.com"
+          git config --local user.name "qywok"
+
+      - name: Pull changes from remote
+        run: |
+          git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
+          (git merge --strategy-option theirs)
+
+      - name: Add and commit changes
+        run: |
+          git add -A 
+          git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+      - name: Push to Hugging Face
+        run: |
+          git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
+
+
+  tebakaja_stock_space-2:
+    name: stock-forecast-svc-2
+    runs-on: ubuntu-latest
+    needs: preprocessing_training
+    environment: Deployment
+
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      SPACE_NAME: tebakaja_stock_space-2
+      HF_USERNAME: tebakaja
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1000
+
+      - name: Check git status
+        run: git status
+
+      - name: Configure git
+        run: |
+          git config --local user.email "alfariqyraihan@gmail.com"
+          git config --local user.name "qywok"
+
+      - name: Pull changes from remote
+        run: |
+          git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
+          (git merge --strategy-option theirs)
+
+      - name: Add and commit changes
+        run: |
+          git add -A 
+          git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+      - name: Push to Hugging Face
+        run: |
+          git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
+
+
+  tebakaja_stock_space-3:
+    name: stock-forecast-svc-3
+    runs-on: ubuntu-latest
+    needs: preprocessing_training
+    environment: Deployment
+
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      SPACE_NAME: tebakaja_stock_space-3
+      HF_USERNAME: tebakaja
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1000
+
+      - name: Check git status
+        run: git status
+
+      - name: Configure git
+        run: |
+          git config --local user.email "alfariqyraihan@gmail.com"
+          git config --local user.name "qywok"
+
+      - name: Pull changes from remote
+        run: |
+          git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
+          (git merge --strategy-option theirs)
+
+      - name: Add and commit changes
+        run: |
+          git add -A 
+          git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+      - name: Push to Hugging Face
+        run: |
+          git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
+
+
+  tebakaja_stock_space-4:
+    name: stock-forecast-svc-4
+    runs-on: ubuntu-latest
+    needs: preprocessing_training
+    environment: Deployment
+
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      SPACE_NAME: tebakaja_stock_space-4
+      HF_USERNAME: tebakaja
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1000
+
+      - name: Check git status
+        run: git status
+
+      - name: Configure git
+        run: |
+          git config --local user.email "alfariqyraihan@gmail.com"
+          git config --local user.name "qywok"
+
+      - name: Pull changes from remote
+        run: |
+          git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
+          (git merge --strategy-option theirs)
+
+      - name: Add and commit changes
+        run: |
+          git add -A 
+          git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+      - name: Push to Hugging Face
+        run: |
+          git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
+
+
+  functional_testing:
+    name: Functional Testing
+    runs-on: ubuntu-latest
+
+    needs:
+      - tebakaja_stock_space-0
+      - tebakaja_stock_space-1
+      - tebakaja_stock_space-2
+      - tebakaja_stock_space-3
+      - tebakaja_stock_space-4
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Testing Proxy Endpoints
+        run: |
+          sleep 60
+          chmod +x endpoints_test.sh && ./endpoints_test.sh
+