init repo

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.csv
+*.env*
+*.pkl
+/app/models/recommendations.py
+/app/routers/mlflow.py
+*.coverage

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.10
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY ./app /code/app
+
+ENV PYTHONPATH "${PYTHONPATH}:/code/app"
+
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8002"]

Pipfile

@@ -0,0 +1,18 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+fastapi = "==0.97.0"
+mlflow = "*"
+boto3 = "*"
+httpx = "*"
+pytest = "*"
+uvicorn = {version = "==0.22.0", extras = ["standard"]}
+coverage = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"

README.md

@@ -1,2 +1,25 @@
-# FlexDR
-Repository of FlexDR application developed within I-NERGY project. 
+# I-NERGY UC7 - FlexDR
+## Repository information
+This is the repository of FlexDR backend application developed within I-NERGY UC7 project.\
+FlexDR service consists of additional services which can be found using the following urls:
+* FlexDR Front-end repository []()
+* FlexDR Orchestration engine []()
+
+## Deployment guidelines
+This section provides instructions to deploy FlexDR FastAPI service and MongoDB locally in docker containers, using Docker Compose.
+[Docker-compose file](docker-compose.yaml) makes use of the environment variables defined in [env file](.env.local) which contains default values for local installation.
+In order to start the containers, run:
+```shell
+docker-compose --env-file=.env.local up --build -d
+```
+
+In order see endpoints documentation users can visit the url: <service_url>:<service_port>/docs
+e.g. http://localhost:8002/docs:
+
+More details on how interact with the application could be found in the documentation section of the repository.
+
+## Run tests
+In order to run the tests:
+1. Start application ([More](#running-application))
+2. Run [script](app/scripts/init_test_db.sh) initialising the database with test data. (Example: ```./init_test_db.sh mongodb ../../mongodb_test_db/flexibility_test```)
+3. Run [script](app/scripts/run_tests.sh) executing tests with coverage (here). This process will take care of cleaning and restoring database prior to tests

app/main.py

@@ -0,0 +1,48 @@
+import os
+from contextlib import asynccontextmanager
+import uvicorn
+from pymongo.database import Database
+from fastapi import FastAPI, Body, Depends, Request, HTTPException
+from pymongo import MongoClient
+from starlette.middleware.cors import CORSMiddleware
+from app.routers import cluster_profiles, ml_models, meters, assignments
+
+MONGO_USER = os.getenv('MONGO_USER')
+MONGO_PASS = os.getenv('MONGO_PASS')
+MONGO_HOST = os.getenv('MONGO_HOST')
+MONGO_PORT = os.getenv('MONGO_PORT')
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    mongo_client: MongoClient = MongoClient(f'mongodb://{MONGO_USER}:{MONGO_PASS}@{MONGO_HOST}:{MONGO_PORT}/')
+    app.db: Database = mongo_client['flexibility']
+    try:
+        yield
+    finally:
+        print("Closing connection")
+        mongo_client.close()
+
+
+app = FastAPI(lifespan=lifespan)
+app.include_router(cluster_profiles.router)
+app.include_router(ml_models.router)
+app.include_router(meters.router)
+app.include_router(assignments.router)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/")
+async def root():
+    return {"message": "Welcome to FlexDR"}
+
+
+if __name__ == "__main__":
+    uvicorn.run("main:app", host="127.0.0.1", port=8000, reload=True)

app/models/assignments.py

@@ -0,0 +1,54 @@
+from pydantic import BaseModel, constr, Field
+from typing import List, Optional
+from bson import ObjectId
+from .common import PyObjectId
+import datetime
+from .ml_models import Cluster, MLModelRes
+from .cluster_profiles import Recommendation
+from .meters import MeterRes
+
+
+class ClusterProfile(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    clusters: List[Cluster]
+    cluster: List[Cluster]
+    name: constr(strip_whitespace=True, min_length=1, max_length=50)
+    short_description: constr(strip_whitespace=True, min_length=1, max_length=300)
+    long_description: constr(strip_whitespace=True, min_length=1, max_length=500)
+    recommendation: Recommendation
+
+
+class Assignment(BaseModel):
+    meter_id: str
+    ml_model_id: str
+    cluster_assigned: int
+    forecasted_load: Optional[List[float]]
+    forecast_date: str
+
+
+class AssignmentUpdate(BaseModel):
+    recommendation: Recommendation
+
+
+class AssignmentRes(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    meter_id: PyObjectId
+    ml_model_id: PyObjectId
+    cluster_assigned: int
+
+    class Config:
+        json_encoders = {ObjectId: str}
+
+
+class AssignmentDetailedRes(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    meter: MeterRes
+    ml_model: MLModelRes
+    assigned_cluster: Optional[int]
+    assigned_cluster_profile: ClusterProfile
+    creation_datetime: datetime.datetime
+    forecast_datetime: datetime.datetime
+    forecasted_load: Optional[List[float]]
+
+    class Config:
+        json_encoders = {ObjectId: str}

app/models/cluster_profiles.py

@@ -0,0 +1,54 @@
+from fastapi.encoders import jsonable_encoder
+from pydantic import BaseModel, constr, Field
+from typing import List, Optional
+from bson import ObjectId
+from .common import PyObjectId
+from .ml_models import Cluster, MLModelRes
+
+
+class Recommendation(BaseModel):
+    name: Optional[constr(strip_whitespace=True, max_length=50)] = Field(default="")
+    description: Optional[constr(strip_whitespace=True, max_length=100)] = Field(default="")
+    details: Optional[constr(strip_whitespace=True, max_length=500)] = Field(default="")
+
+
+# used for cluster profile creation
+class ClusterProfile(BaseModel):
+    # id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
+    ml_model_id: str
+    selected_clusters: List[int]
+    name: constr(strip_whitespace=True, min_length=1, max_length=50)
+    short_description: constr(strip_whitespace=True, min_length=1, max_length=300)
+    long_description: constr(strip_whitespace=True, min_length=1, max_length=900)
+    recommendation: Recommendation
+
+
+# used for cluster profile update
+class ClusterProfileUpdate(BaseModel):
+    # id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
+    # ml_model_id: str
+    selected_clusters: Optional[List[int]]
+    name: constr(strip_whitespace=True, min_length=1, max_length=50)
+    short_description: Optional[constr(strip_whitespace=True, min_length=1, max_length=300)]
+    long_description: Optional[constr(strip_whitespace=True, min_length=1, max_length=500)]
+    recommendation: Optional[Recommendation]
+
+    def to_dict(self):
+        model_dict = jsonable_encoder(self)
+        # Filter out keys with None values
+        filtered_dict = {key: value for key, value in model_dict.items() if value is not None}
+        return filtered_dict
+
+
+# used for response
+class ClusterProfileRes(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    ml_model: MLModelRes
+    clusters: List[Cluster]
+    name: constr(strip_whitespace=True, min_length=1, max_length=50)
+    short_description: constr(strip_whitespace=True, min_length=1, max_length=300)
+    long_description: constr(strip_whitespace=True, min_length=1, max_length=500)
+    recommendation: Recommendation
+
+    class Config:
+        json_encoders = {ObjectId: str}

app/models/common.py

@@ -0,0 +1,24 @@
+from typing import List
+from bson import ObjectId
+from pydantic import BaseModel
+
+
+class PyObjectId(ObjectId):
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        if not ObjectId.is_valid(v):
+            raise ValueError("Invalid objectid")
+        return ObjectId(v)
+
+    @classmethod
+    def __modify_schema__(cls, field_schema):
+        field_schema.update(type="string")
+
+
+class LineGraphData(BaseModel):
+    x: List[float]
+    y: List[float]

app/models/load_profiles.py

@@ -0,0 +1,76 @@
+from pydantic import BaseModel, constr, Field
+from typing import List, Optional
+from bson import ObjectId
+from .common import PyObjectId
+
+
+class TimeRange(BaseModel):
+    start_time: str
+    end_time: str
+
+
+class LineGraphData(BaseModel):
+    x: List[float]
+    y: List[float]
+
+
+class ClusterCreate(BaseModel):
+    number: int
+    image: str
+
+
+# TODO prevent modification of number and image. Only append or delete.
+class ClusterUpdate(BaseModel):
+    number: int
+    image: str
+
+
+# used for creation
+class LoadProfileModel(BaseModel):
+    # id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
+    name: constr(strip_whitespace=True, min_length=1, max_length=50)
+    short_description: constr(strip_whitespace=True, max_length=300, min_length=1)
+    long_description: constr(strip_whitespace=True, min_length=1, max_length=500)
+    number: int
+    clusters: List[ClusterCreate]
+    recommendation: Optional[constr(strip_whitespace=True, min_length=1, max_length=500)]
+    # time_ranges: List[TimeRange]
+    # line_graph_data: List[LineGraphData]
+
+    # class Config:
+    #     allow_population_by_field_name = True
+    #     arbitrary_types_allowed = True
+    #     json_encoders = {ObjectId: str}
+
+
+# used for response
+class LoadProfileModelBase(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    name: str
+    short_description: str
+    long_description: str
+    number: int
+    clusters: Optional[List[ClusterCreate]]
+    recommendation: Optional[str]
+
+    # time_ranges: List[TimeRange]
+    # line_graph_data: List[LineGraphData]
+
+    class Config:
+        json_encoders = {ObjectId: str}
+
+
+# used for update
+class LoadProfileUpdateModel(BaseModel):
+    # id: PyObjectId = Field(..., alias="_id")
+    name: Optional[constr(strip_whitespace=True, min_length=1, max_length=50)]
+    short_description: Optional[constr(strip_whitespace=True, max_length=50, min_length=1)]
+    long_description: Optional[constr(strip_whitespace=True, min_length=1)]
+    number: Optional[int]
+    clusters: List[ClusterUpdate]
+    recommendation: Optional[constr(strip_whitespace=True, min_length=1, max_length=500)]
+
+    class Config:
+        allow_population_by_field_name = True
+        arbitrary_types_allowed = True
+        json_encoders = {ObjectId: str}

app/models/meters.py

@@ -0,0 +1,46 @@
+from typing import Optional
+from fastapi.encoders import jsonable_encoder
+from pydantic import BaseModel, constr, Field
+from bson import ObjectId
+from .common import PyObjectId
+
+
+# used for creation
+class Meter(BaseModel):
+    device_id: constr(strip_whitespace=True, min_length=1, max_length=50)
+    contract_pw: float
+    prod_pw: float
+    type: constr(strip_whitespace=True, min_length=1, max_length=100)
+
+
+# used for response
+class MeterRes(BaseModel):
+    id: PyObjectId = Field(..., alias="_id")
+    device_id: constr(strip_whitespace=True, min_length=1, max_length=50)
+    contract_pw: Optional[float]
+    prod_pw: Optional[float]
+    type: constr(strip_whitespace=True, min_length=1, max_length=100)
+
+    class Config:
+        json_encoders = {ObjectId: str}
+
+
+# used for update
+class MeterUpdate(BaseModel):
+    # id: PyObjectId = Field(..., alias="_id")
+    device_id: Optional[constr(strip_whitespace=True, min_length=1, max_length=50)]
+    contract_pw: Optional[float]
+    prod_pw: Optional[float]
+    type: Optional[constr(strip_whitespace=True, min_length=1, max_length=100)]
+
+    class Config:
+        allow_population_by_field_name = True
+        arbitrary_types_allowed = True
+        json_encoders = {ObjectId: str}
+
+    # remove None values
+    def to_dict(self):
+        model_dict = jsonable_encoder(self)
+        # Filter out keys with None values
+        filtered_dict = {key: value for key, value in model_dict.items() if value is not None}
+        return filtered_dict