Merge pull request #283 from techman83/refactor/queue_handler

Refactor Game Management
KSP-CKAN · Apr 6, 2023 · 1b6e52e · 1b6e52e
2 parents 907a196 + 29805b6
commit 1b6e52e
Show file tree

Hide file tree

Showing 40 changed files with 1,677 additions and 390 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -11,6 +11,6 @@
 		// uses the right one for terminals and tasks. For example, /bin/bash (or /bin/ash for Alpine).
 		"terminal.integrated.shell.linux": null
 	},
-	"postCreateCommand": "pip install --user -e netkan/.",
+	"postCreateCommand": "pip install --user -e netkan/.['development']",
 	"extensions": []
 }
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -8,18 +8,18 @@ services:
       target: dev
     environment:
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      CKANMETA_REMOTE: ${CKAN_METADATA_PATH}
-      CKANMETA_USER: ${CKAN_METADATA_USER}
-      CKANMETA_REPO: ${CKAN_METADATA_REPO}
-      NETKAN_REMOTE: ${NETKAN_METADATA_PATH}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
+      CKAN_USER: ${CKAN_METADATA_USER}
+      CKAN_REPOS: ${CKAN_METADATA_REPOS}
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
       GH_Token: ${CKAN_GH_Token}
       SQS_TIMEOUT: 30
       STATUS_DB: DevNetKANStatus
       XKAN_GHSECRET: test
-      INFLATION_SQS_QUEUE: InboundDev.fifo
+      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
       MIRROR_SQS_QUEUE: MirroringDev.fifo
       STATUS_BUCKET: ckan-test-status
       STATUS_INTERVAL: 0

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -0,0 +1,30 @@
+name: Coverage
+
+on:
+  - pull_request
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: netkan
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install test dependencies
+        run: pip install .[test]
+      - name: force our git config
+        run: cp .gitconfig ~/.
+      - name: Run Coverage
+        run: coverage run -m pytest
+      - name: Generate report
+        run: coverage xml
+      - name: Get Cover
+        uses: orgoro/coverage@v3
+        with:
+          coverageFile: netkan/coverage.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,5 @@ __pycache__
 .pytest_cache/
 .eggs
 netkan/build/
+coverage.xml
+.coverage
diff --git a/dev-stack.py b/dev-stack.py
@@ -20,10 +20,14 @@
 
 t.set_description("Generate NetKAN Infrastructure CF Template")
 
-inbound = t.add_resource(Queue("InboundDev",
-                               QueueName="InboundDev.fifo",
-                               ReceiveMessageWaitTimeSeconds=20,
-                               FifoQueue=True))
+inbound_ksp = t.add_resource(Queue("InboundDevKsp",
+                                   QueueName="InboundDevKsp.fifo",
+                                   ReceiveMessageWaitTimeSeconds=20,
+                                   FifoQueue=True))
+inbound_ksp2 = t.add_resource(Queue("InboundDevKsp2",
+                                    QueueName="InboundDevKsp2.fifo",
+                                    ReceiveMessageWaitTimeSeconds=20,
+                                    FifoQueue=True))
 outbound = t.add_resource(Queue("OutboundDev",
                                 QueueName="OutboundDev.fifo",
                                 ReceiveMessageWaitTimeSeconds=20,
@@ -56,7 +60,8 @@
                     "sqs:GetQueueAttributes",
                 ],
                 "Resource": [
-                    GetAtt(inbound, "Arn"),
+                    GetAtt(inbound_ksp, "Arn"),
+                    GetAtt(inbound_ksp2, "Arn"),
                     GetAtt(outbound, "Arn"),
                     GetAtt(addqueue, "Arn"),
                     GetAtt(mirrorqueue, "Arn"),
@@ -71,7 +76,7 @@
     }
 ))
 
-for queue in [inbound, outbound, addqueue, mirrorqueue]:
+for queue in [inbound_ksp, inbound_ksp2, outbound, addqueue, mirrorqueue]:
     t.add_output([
         Output(
             "{}QueueURL".format(queue.title),

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,9 +10,9 @@ services:
       target: dev
     environment:
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      CKANMETA_REMOTE: ${CKAN_METADATA_PATH}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
       CKANMETA_USER: ${CKAN_METADATA_USER}
-      CKANMETA_REPO: ${CKAN_METADATA_REPO}
+      CKANMETA_REPOS: ${CKAN_METADATA_REPOS}
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
@@ -30,13 +30,33 @@ services:
       context: netkan/.
       target: dev
     environment:
-      NETKAN_REMOTE: ${NETKAN_METADATA_PATH}
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      CKANMETA_REMOTE: ${CKAN_METADATA_PATH}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
-      SQS_QUEUE: InboundDev.fifo
+      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
+      GAME_ID: ksp2
+      MAX_QUEUED: 1
+      DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
+      DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
+    volumes:
+      - ./netkan:/home/netkan/netkan
+    command: scheduler --dev
+  scheduler:
+    build:
+      context: netkan/.
+      target: dev
+    environment:
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
+      SSH_KEY: ${CKAN_NETKAN_SSHKEY}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
+      AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
+      AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
+      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
+      GAME_ID: ksp2
       MAX_QUEUED: 1
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
@@ -46,7 +66,8 @@ services:
   inflator:
     image: kspckan/inflator
     environment:
-      QUEUES: InboundDev.fifo,OutboundDev.fifo
+      QUEUES: InboundDevKsp.fifo,OutboundDev.fifo
+      GAME: KSP
       GH_Token: ${CKAN_GH_Token}
       AWS_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
@@ -57,18 +78,20 @@ services:
       # distros is 1000:1000, which will match the
       # user in the container.
       - ${HOME}/ckan_cache:/home/netkan/ckan_cache
-  legacyhooks:
-    image: kspckan/webhooks
+  inflator-ksp2:
+    image: kspckan/inflator
     environment:
-      SSH_KEY: ${CKAN_NETKAN_SSHKEY}
+      QUEUES: InboundDevKsp2.fifo,OutboundDev.fifo
+      GAME: KSP2
       GH_Token: ${CKAN_GH_Token}
-      XKAN_GHSECRET: test
-      CKAN_meta: ${CKAN_METADATA_PATH}
-      NetKAN: ${NETKAN_METADATA_PATH}
-      IA_access: test
-      IA_secret: test
-      IA_collection: test
+      AWS_REGION: ${CKAN_AWS_DEFAULT_REGION}
+      AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
     volumes:
+      # Docker will create this with root perms if
+      # it's not created first. First user on most
+      # distros is 1000:1000, which will match the
+      # user in the container.
       - ${HOME}/ckan_cache:/home/netkan/ckan_cache
   webhooks:
     build:
@@ -77,14 +100,17 @@ services:
     environment:
       XKAN_GHSECRET: test
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      NETKAN_REMOTE: ${NETKAN_METADATA_PATH}
-      INFLATION_SQS_QUEUE: InboundDev.fifo
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
+      INFLATION_SQS_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
       MIRROR_SQS_QUEUE: MirroringDev.fifo
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
+      CKANMETA_USER: ${CKAN_METADATA_USER}
+      CKANMETA_REPOS: ${CKAN_METADATA_REPOS}
     entrypoint: .local/bin/gunicorn
     command: -b 0.0.0.0:5000 --access-logfile - "netkan.webhooks:create_app()"
   adder:
@@ -93,7 +119,7 @@ services:
       target: dev
     environment:
       SQS_QUEUE: AddingDev.info
-      NETKAN_REMOTE: ${NETKAN_METADATA_PATH}
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
       NETKAN_USER: ${CKAN_NETKAN_USER}
       NETKAN_REPO:  ${CKAN_NETKAN_REPO}
     command: spacedock-adder
@@ -107,7 +133,7 @@ services:
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
-      CKANMETA_REMOTE: ${CKAN_METADATA_PATH}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
       IA_access: test
       IA_secret: test
       IA_collection: test
@@ -162,8 +188,8 @@ services:
       target: dev
     environment:
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      NETKAN_REMOTE: ${NETKAN_METADATA_PATH}
-      CKANMETA_REMOTE: ${CKAN_METADATA_PATH}
+      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
+      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
       GH_Token: ${CKAN_GH_Token}
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}

diff --git a/netkan/.coveragerc b/netkan/.coveragerc
@@ -0,0 +1,7 @@
+[report]
+include =
+    netkan/*
+
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING: