Plural Cloud CRUD (#1328)

Co-authored-by: Jake Laderman <jsladerman@gmail.com>

.github/workflows/test.yaml

@@ -152,6 +152,7 @@ jobs:
           curl -sL 'https://github.com/pluralsh/plural-cli/releases/download/v0.5.18/plural-cli_0.5.18_Linux_amd64.tar.gz' | tar xzvf -
           chmod +x plural
           cp plural /usr/local/bin/plural
+      - run: make install-cockroach
       - run: make testup
       - name: Restore dependencies cache
         uses: actions/cache@v3

.github/workflows/www.yaml

@@ -136,41 +136,41 @@ jobs:
           node-version: ${{ steps.engines.outputs.nodeVersion }}
       - run: yarn --immutable
       - run: yarn lint
-  e2e:
-    name: End-to-end test
-    runs-on: ubuntu-20.04
-    env:
-      CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }}
-      CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: www
-    steps:
-      - name: 'Checkout'
-        uses: actions/checkout@v3
-      - name: Read Node.js version from package.json
-        run: echo ::set-output name=nodeVersion::$(node -p "require('./package.json').engines.node")
-        id: engines
-      - name: 'Setup Node'
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ steps.engines.outputs.nodeVersion }}
-      - run: yarn # Should run the --immutable in the CI by default
-      - run: cd e2e && yarn
-      - run: yarn e2e
-      - uses: 8398a7/action-slack@v3
-        if: failure()
-        with:
-          status: ${{ job.status }}
-          fields: workflow,repo,commit,author,pullRequest
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CYPRESS_WEBHOOK }}
-      - name: Upload Screenshots and Videos to Slack
-        if: failure()
-        uses: trymbill/cypress-slack-video-upload-action@v1.3.0
-        with:
-          token: ${{ secrets.SLACK_CYPRESS_TOKEN }}
-          workdir: www/e2e/cypress
-          channels: cypress-artifacts
-          message-text: "See the attached videos and screenshots for more information."
+  # e2e:
+  #   name: End-to-end test
+  #   runs-on: ubuntu-20.04
+  #   env:
+  #     CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }}
+  #     CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }}
+  #   defaults:
+  #     run:
+  #       shell: bash
+  #       working-directory: www
+  #   steps:
+  #     - name: 'Checkout'
+  #       uses: actions/checkout@v3
+  #     - name: Read Node.js version from package.json
+  #       run: echo ::set-output name=nodeVersion::$(node -p "require('./package.json').engines.node")
+  #       id: engines
+  #     - name: 'Setup Node'
+  #       uses: actions/setup-node@v3
+  #       with:
+  #         node-version: ${{ steps.engines.outputs.nodeVersion }}
+  #     - run: yarn # Should run the --immutable in the CI by default
+  #     - run: cd e2e && yarn
+  #     - run: yarn e2e
+  #     - uses: 8398a7/action-slack@v3
+  #       if: failure()
+  #       with:
+  #         status: ${{ job.status }}
+  #         fields: workflow,repo,commit,author,pullRequest
+  #       env:
+  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CYPRESS_WEBHOOK }}
+  #     - name: Upload Screenshots and Videos to Slack
+  #       if: failure()
+  #       uses: trymbill/cypress-slack-video-upload-action@v1.3.0
+  #       with:
+  #         token: ${{ secrets.SLACK_CYPRESS_TOKEN }}
+  #         workdir: www/e2e/cypress
+  #         channels: cypress-artifacts
+  #         message-text: "See the attached videos and screenshots for more information."

.gitignore

@@ -67,4 +67,7 @@ yarn-error.log*
 cert.pem
 key.pem
 
+
+/test-certs/
+
 .vscode

.tool-versions

@@ -1,2 +1,2 @@
-erlang 24.3.4.14
-elixir 1.12.3
+erlang 24.3.4.17
+elixir 1.13.4

Dockerfile

@@ -1,4 +1,4 @@
-FROM bitwalker/alpine-elixir:1.12.3 AS builder
+FROM bitwalker/alpine-elixir:1.13.4 AS builder
 
 # The following are build arguments used to change variable parts of the image.
 # The name of your application/release (required)

Makefile

@@ -8,6 +8,7 @@ DKR_HOST ?= dkr.plural.sh
 dep ?= forge-core
 GIT_COMMIT ?= abe123
 TARGETARCH ?= amd64
+COCKROACH_VSN ?= v24.1.3
 
 help:
 	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -48,11 +49,26 @@ else
 	docker push $(DKR_HOST)/plural/${APP_NAME}:$(APP_VSN)
 endif
 
-testup: ## sets up dependent services for test
-	docker-compose up -d
+install-cockroach:
+	sudo curl https://binaries.cockroachdb.com/cockroach-$(COCKROACH_VSN).linux-amd64.tgz | tar -xz && \
+	sudo cp -i cockroach-$(COCKROACH_VSN).linux-amd64/cockroach /usr/local/bin/ && \
+	sudo mkdir -p /usr/local/lib/cockroach && \
+	sudo cp -i cockroach-$(COCKROACH_VSN).linux-amd64/lib/libgeos.so /usr/local/lib/cockroach/ && \
+	sudo cp -i cockroach-$(COCKROACH_VSN).linux-amd64/lib/libgeos_c.so /usr/local/lib/cockroach/ && \
+	cockroach version
+
+test-certs:
+	mkdir test-certs && \
+	cockroach cert create-ca --certs-dir test-certs --ca-key test-certs/ca.key && \
+	cockroach cert create-node localhost 127.0.0.1 --certs-dir test-certs --ca-key test-certs/ca.key && \
+	cockroach cert create-client root --certs-dir test-certs --ca-key test-certs/ca.key && \
+	cockroach cert list --certs-dir test-certs
+
+testup: test-certs ## sets up dependent services for test
+	docker compose up -d
 
 testdown: ## tear down test dependencies
-	docker-compose down
+	docker compose down
 
 connectdb: ## proxies the db in kubernetes via kubectl
 	@echo "run psql -U forge -h 127.0.0.1 forge to connect"

apps/core/lib/core/clients/console.ex

@@ -0,0 +1,96 @@
+defmodule Core.Clients.Console do
+  require Logger
+
+  @clusters_q """
+  query {
+    clusters(first: 100) {
+      edges { node { name id distro metadata } }
+    }
+  }
+  """
+
+  @create_svc_q """
+  mutation Create($clusterId: ID!, $attributes: ServiceDeploymentAttributes!) {
+    createServiceDeployment(clusterId: $clusterId, attributes: $attributes) {
+      id
+    }
+  }
+  """
+
+  @delete_svc_q """
+  mutation Delete($id: ID!) {
+    deleteServiceDeployment(id: $id) {
+      id
+    }
+  }
+  """
+
+  @update_svc_q """
+  mutation Update($id: ID!, $attributes: ServiceUpdateAttributes!) {
+    updateServiceDeployment(id: $id) {
+      id
+    }
+  }
+  """
+
+  @repo_q """
+  query Repo($url: String!) {
+    gitRepository(url: $url) {
+      id
+    }
+  }
+  """
+
+  def new(url, token) do
+    Req.new(base_url: url, auth: "Token #{token}")
+    |> AbsintheClient.attach()
+  end
+
+  def clusters(client) do
+    Req.post(client, graphql: @clusters_q)
+    |> case do
+      {:ok, %Req.Response{body: %{"clusters" => %{"edges" => edges}}}} -> {:ok, Enum.map(edges, & &1["node"])}
+      res ->
+        Logger.warn "Failed to fetch clusters: #{inspect(res)}"
+        {:error, "could not fetch clusters"}
+    end
+  end
+
+  def repo(client, url) do
+    Req.post(client, graphql: {@repo_q, %{url: url}})
+    |> case do
+      {:ok, %Req.Response{body: %{"gitRepository" => %{"id" => id}}}} -> {:ok, id}
+      res ->
+        Logger.warn "Failed to fetch clusters: #{inspect(res)}"
+        {:error, "could not fetch repo"}
+    end
+  end
+
+  def create_service(client, cluster_id, attrs) do
+    Req.post(client, graphql: {@create_svc_q, %{clusterId: cluster_id, attributes: attrs}})
+    |> service_resp("createServiceDeployment")
+  end
+
+  def update_service(client, id, attrs) do
+    Req.post(client, graphql: {@update_svc_q, %{id: id, attributes: attrs}})
+    |> service_resp("updateServiceDeployment")
+  end
+
+  def delete_service(client, id) do
+    Req.post(client, graphql: {@delete_svc_q, %{id: id}})
+    |> service_resp("deleteServiceDeployment")
+  end
+
+  defp service_resp({:ok, %Req.Response{status: 200, body: body}}, field) do
+    case body[field] do
+      %{"id" => id} -> {:ok, id}
+      err ->
+        Logger.warn "invalid console gql response: #{inspect(err)}"
+    end
+  end
+
+  defp service_resp(resp, _) do
+    Logger.error "failed to fetch from console: #{inspect(resp)}"
+    {:error, "console error"}
+  end
+end

apps/core/lib/core/conduit/base.ex

@@ -13,6 +13,7 @@ defmodule Core.Conduit.Base do
         defqueue "plural.upgrade"
         defqueue "plural.scan"
         defqueue "plural.cluster"
+        defqueue "plural.cloud"
       end
 
       pipeline :out_tracking do
@@ -36,6 +37,7 @@ defmodule Core.Conduit.Base do
         publish :upgrade, exchange: "plural.topic", to: "plural.upgrade"
         publish :scan, exchange: "plural.topic", to: "plural.scan"
         publish :cluster, exchange: "plural.topic", to: "plural.cluster"
+        publish :cloud, exchange: "plural.topic", to: "plural.cloud"
       end
 
       outgoing do

apps/core/lib/core/policies/cloud.ex

@@ -0,0 +1,16 @@
+defmodule Core.Policies.Cloud do
+  use Piazza.Policy
+  alias Core.Schema.{User, ConsoleInstance}
+  alias Core.Services.Payments
+
+  def can?(%User{} = user, %ConsoleInstance{}, :create) do
+    case Payments.has_feature?(user, :cd) do
+      true -> :pass
+      _ -> {:error, "you must be on a paid plan to use Plural Cloud"}
+    end
+  end
+
+  def can?(u, %Ecto.Changeset{} = cs, action), do: can?(u, apply_changes(cs), action)
+
+  def can?(_, _, _), do: :pass
+end

apps/core/lib/core/pubsub/events.ex

@@ -90,3 +90,8 @@ defmodule Core.PubSub.ClusterDependencyCreated, do: use Piazza.PubSub.Event
 defmodule Core.PubSub.DeferredUpdateCreated, do: use Piazza.PubSub.Event
 
 defmodule Core.PubSub.UpgradesPromoted, do: use Piazza.PubSub.Event
+
+defmodule Core.PubSub.ConsoleInstanceCreated, do: use Piazza.PubSub.Event
+defmodule Core.PubSub.ConsoleInstanceUpdated, do: use Piazza.PubSub.Event
+defmodule Core.PubSub.ConsoleInstanceDeleted, do: use Piazza.PubSub.Event
+defmodule Core.PubSub.ConsoleInstanceReaped, do: use Piazza.PubSub.Event

apps/core/lib/core/pubsub/protocols/fanout.ex

@@ -235,3 +235,14 @@ defimpl Core.PubSub.Fanout, for: [Core.PubSub.RoleCreated, Core.PubSub.RoleUpdat
     |> Enum.count()
   end
 end
+
+defimpl Core.PubSub.Fanout, for: [
+    Core.PubSub.ConsoleInstanceCreated,
+    Core.PubSub.ConsoleInstanceUpdated,
+    Core.PubSub.ConsoleInstanceDeleted
+  ] do
+  def fanout(event) do
+    %Conduit.Message{body: event}
+    |> Core.Conduit.Broker.publish(:cloud)
+  end
+end