[Cal-4] Calendar microservice (#357)

* split the calendar into its own submodule
* split ScrapeRoomTask into its own module
* added a calendar deployment
* improved the documentation on how to get the calendar up and running

.github/workflows/calendar.yml

@@ -0,0 +1,50 @@
+name: calendar CD
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    # Set job outputs to values from filter step
+    outputs:
+      calendar: ${{ steps.filter.outputs.calendar }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            calendar:
+              - '.github/**'
+              - 'calendar/**'
+  calendar-build:
+    needs:
+      - changes
+    if: ${{ needs.changes.outputs.calendar == 'true' }}
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      image_suffix: calendar
+      context: ./calendar
+      dockerfile: Dockerfile
+    permissions:
+      contents: read
+      packages: write
+  calendar-deployment:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs:
+      - calendar-build
+    steps:
+      - run: curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64 && chmod +x /usr/local/bin/argocd
+      - run: argocd app actions run navigatum-prod restart --kind Deployment --resource-name calendar --auth-token ${{ secrets.ARGOCD_TOKEN }} --server ${{ secrets.ARGOCD_SERVER }}
+  calendar-staging-deployment:
+    if: github.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs:
+      - calendar-build
+    steps:
+      - run: curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64 && chmod +x /usr/local/bin/argocd
+      - run: argocd app actions run pr-${{github.event.number}} restart --kind Deployment --resource-name calendar --auth-token ${{ secrets.ARGOCD_TOKEN }} --server ${{ secrets.ARGOCD_SERVER }}

README.md

@@ -46,6 +46,7 @@ For an overview of how the components work, have a look at the
 - `data/` contains the code to obtain and process the data
 - `server/` contains the API server written in Rust, including MeiliSearch as a search backend
 - `feedback/` contains the feedback-API server written in Rust
+- `calendar/` contains the calendar-API server written in Rust
 - `webclient/` contains a JS based web-frontend for the API
 - `deployment/` contains deployment related configuration
 - `map/` contains information about our own map, how to style it and how to run it

calendar/.dockerignore

@@ -0,0 +1,4 @@
+target
+.hypothesis/
+Cargo.lock
+data

calendar/.gitignore

@@ -0,0 +1,3 @@
+/target
+Cargo.lock
+data/*

calendar/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "navigatum-calendar"
+version = "1.0.0"
+authors = ["Markus A <ge75sig@mytum.de>", "Frank Elsinga <frank@elsinga.de>"]
+edition = "2021"
+description = "Navigating around TUM with excellence – An API and website to search for rooms, buildings and other places"
+repository = "https://github.com/TUM-Dev/navigatum"
+readme = "README.md"
+license = "GPL-3.0"
+keywords = ["website", "navigation", "api-rest", "tum"]
+
+[[bin]]
+name = "navigatum-calendar"
+path = "src/main.rs"
+
+[profile.release]
+strip = true
+
+[dependencies]
+log = "0.4.17"
+env_logger = "0.10.0"
+diesel = { version = "2.0.2", features = ["default","chrono","sqlite"] }
+actix-web = "4.2.1"
+actix-rt = "2.7.0"
+rustls = "0.20.7"
+awc = { version= "3.0.1", features = ["rustls"] }
+serde = { version = "1.0.148", features = ["derive"] }
+serde_json = "1.0.89"
+actix-cors = "0.6.4"
+tokio = { version = "1.22.0", features = ["full"] }
+futures = "0.3.25"
+chrono = { version="0.4.23", features=["serde","rustc-serialize"] }
+minidom = "0.15.0"
+rand = "0.8.5"

calendar/Dockerfile

@@ -0,0 +1,28 @@
+# Compile
+FROM    rust:1.66-alpine AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev sqlite-dev
+
+WORKDIR /nav
+COPY    ./Cargo.* ./
+COPY    ./src ./src
+RUN     RUSTFLAGS="-C target-feature=-crt-static" cargo build --release
+
+
+# RUN
+FROM alpine:3.17
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc sqlite-libs tini
+
+# add `navigatum-calendar` to the `/bin` so we can run it from anywhere and it's easy to find.
+COPY    --from=compiler /nav/target/release/navigatum-calendar /bin/navigatum-calendar
+
+ARG     GIT_COMMIT_SHA
+ENV     GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
+
+EXPOSE  8083
+
+ENTRYPOINT ["tini", "--"]
+HEALTHCHECK --start-period=20m  --timeout=10s CMD curl --fail localhost:8083/api/health || exit 1
+CMD     /bin/navigatum-calendar

calendar/README.md

@@ -0,0 +1,97 @@
+# Calendar
+
+This folder contains the calendar-API server for NavigaTUM.
+
+This is separated from the server because:
+
+- it has virtually no shared dependencies (natural fault line)
+- this way, we can deploy the calendar-API independently of the main server.
+  The Reason why this is important is, that scraping calendar entries is expensive for TUMOnline.
+  => We have to be resourcefully and can't waste this scraped state by redeploying at will
+  => Making this a StatefulSet instead of a Deployment makes sense
+
+## Getting started
+
+### Prerequisites
+
+For getting started, there are some system dependencys which you will need.
+Please follow the [system dependencys docs](resources/documentation/Dependencys.md) before trying to run this part of our project.
+
+### How to Set up the Sqlite Database (needed for the `get`, `legacy_redirect` and `preview` endpoints)
+
+#### Getting the data
+
+To populate the database, you will need to get said data.
+There are multiple ways to do this, but the easiest way is to download the data from our [website](https://nav.tum.de/).
+
+(Assuming you are in the `server` directory)
+
+```bash
+mkdir -p data
+wget -P data https://nav.tum.de/cdn/api_data.json
+```
+
+Alternatively, you can run the `data` part of this project and generate this file by that part of our docs.
+To link the output directory to the server data directory, so that you don't need to copy on every update you can use:
+
+```bash
+ln -s ../data/output data
+```
+
+#### Setting up the database
+
+To set up the database, you will need to run the `load_api_data_to_db.py` script:
+
+```bash
+python3 load_api_data_to_db.py
+```
+
+### Starting the server
+
+Run `cargo run` to start the server.
+The server should now be available on `localhost:8081`.
+
+Note that `cargo run --release` is used to start the server for an optimised production build (use this if you want to profile performance, it makes quite a difference).
+
+### API-Changes
+
+#### Editing
+
+If you have made changes to the API, you need to update the API documentation.
+
+There are two editors for the API documentation (both are imperfect):
+
+- [Swagger Editor](https://editor.swagger.io/?url=https://raw.githubusercontent.com/TUM-Dev/navigatum/main/openapi.yaml)
+- [stoplight](stoplight.io)
+
+#### Testing
+
+Of course documentation is one part of the process. If the changes are substantial, you should also run an API-Fuzz-Test:
+To make sure that this specification is up-to-date and without holes, we run [schemathesis](https://github.com/schemathesis/schemathesis) using the following command on API Server:
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip install schemathesis
+st run --workers=auto --base-url=http://localhost:8081 --checks=all ../openapi.yaml
+```
+
+Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and some fuzzing-goals are automatically tested in our CI.  
+You can exchange `--base-url=http://localhost:8081` to `--base-url=https://nav.tum.sexy` for the full public API, or restrict your scope using a option like `--endpoint=/api/calendar/`.
+
+## License
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+---

calendar/diesel.toml

@@ -0,0 +1,8 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"
+
+[migrations_directory]
+dir = "migrations"

calendar/src/main.rs

@@ -0,0 +1,55 @@
+mod schema;
+mod scraping;
+mod utils;
+
+use actix_cors::Cors;
+use actix_web::{get, middleware, web, App, HttpResponse, HttpServer};
+
+const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB
+
+#[get("/api/calendar/source_code")]
+async fn source_code_handler() -> HttpResponse {
+    let gh_base = "https://github.com/TUM-Dev/navigatum".to_string();
+    let commit_hash = std::env::var("GIT_COMMIT_SHA");
+    let github_link = match commit_hash {
+        Ok(hash) => format!("{gh_base}/tree/{hash}"),
+        Err(_) => gh_base,
+    };
+    HttpResponse::Ok()
+        .content_type("text/plain")
+        .body(github_link)
+}
+
+#[get("/api/calendar/health")]
+async fn health_handler() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("text/plain")
+        .body("healthy")
+}
+
+#[actix_web::main]
+async fn main() -> std::io::Result<()> {
+    env_logger::init_from_env(env_logger::Env::default().default_filter_or("info"));
+
+    actix_rt::spawn(async move {
+        scraping::continous_scraping::start_scraping().await;
+    });
+    HttpServer::new(move || {
+        let cors = Cors::default()
+            .allow_any_origin()
+            .allow_any_header()
+            .allowed_methods(vec!["GET"])
+            .max_age(3600);
+
+        App::new()
+            .wrap(cors)
+            .wrap(middleware::Logger::default().exclude("/api/calendar/health"))
+            .wrap(middleware::Compress::default())
+            .app_data(web::JsonConfig::default().limit(MAX_JSON_PAYLOAD))
+            .service(source_code_handler)
+            .service(health_handler)
+    })
+    .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:8060".to_string()))?
+    .run()
+    .await
+}

calendar/src/schema.rs

@@ -0,0 +1,83 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    calendar (single_event_id) {
+        key -> Text,
+        dtstart -> Timestamp,
+        dtend -> Timestamp,
+        dtstamp -> Timestamp,
+        event_id -> Integer,
+        event_title -> Text,
+        single_event_id -> Integer,
+        single_event_type_id -> Text,
+        single_event_type_name -> Text,
+        event_type_id -> Text,
+        event_type_name -> Nullable<Text>,
+        course_type_name -> Nullable<Text>,
+        course_type -> Nullable<Text>,
+        course_code -> Nullable<Text>,
+        course_semester_hours -> Nullable<Integer>,
+        group_id -> Nullable<Text>,
+        xgroup -> Nullable<Text>,
+        status_id -> Text,
+        status -> Text,
+        comment -> Text,
+    }
+}
+
+diesel::table! {
+    calendar_scrape (single_event_id) {
+        key -> Text,
+        dtstart -> Timestamp,
+        dtend -> Timestamp,
+        dtstamp -> Timestamp,
+        event_id -> Integer,
+        event_title -> Text,
+        single_event_id -> Integer,
+        single_event_type_id -> Text,
+        single_event_type_name -> Text,
+        event_type_id -> Text,
+        event_type_name -> Nullable<Text>,
+        course_type_name -> Nullable<Text>,
+        course_type -> Nullable<Text>,
+        course_code -> Nullable<Text>,
+        course_semester_hours -> Nullable<Integer>,
+        group_id -> Nullable<Text>,
+        xgroup -> Nullable<Text>,
+        status_id -> Text,
+        status -> Text,
+        comment -> Text,
+    }
+}
+
+diesel::table! {
+    de (key) {
+        key -> Text,
+        name -> Text,
+        tumonline_room_nr -> Nullable<Integer>,
+        arch_name -> Nullable<Text>,
+        #[sql_name = "type"]
+        type_ -> Text,
+        type_common_name -> Text,
+        lat -> Float,
+        lon -> Float,
+        data -> Text,
+    }
+}
+
+diesel::table! {
+    en (key) {
+        key -> Text,
+        name -> Text,
+        tumonline_room_nr -> Nullable<Integer>,
+        arch_name -> Nullable<Text>,
+        #[sql_name = "type"]
+        type_ -> Text,
+        type_common_name -> Text,
+        lat -> Float,
+        lon -> Float,
+        data -> Text,
+    }
+}
+
+diesel::allow_tables_to_appear_in_same_query!(calendar, calendar_scrape, de, en,);