Add auto-deprovisioning crons

Sends first and second warning emails, then deletes

Makefile

@@ -57,19 +57,18 @@ install-cockroach:
 	sudo cp -i cockroach-$(COCKROACH_VSN).linux-amd64/lib/libgeos_c.so /usr/local/lib/cockroach/ && \
 	cockroach version
 
-setup-certs:
+test-certs:
 	mkdir test-certs && \
 	cockroach cert create-ca --certs-dir test-certs --ca-key test-certs/ca.key && \
 	cockroach cert create-node localhost 127.0.0.1 --certs-dir test-certs --ca-key test-certs/ca.key && \
 	cockroach cert create-client root --certs-dir test-certs --ca-key test-certs/ca.key && \
 	cockroach cert list --certs-dir test-certs
 
-testup: setup-certs ## sets up dependent services for test
+testup: test-certs ## sets up dependent services for test
 	docker compose up -d
 
 testdown: ## tear down test dependencies
-	docker compose down && \
-	  rm -rf test-certs || echo 'no cert folder to remove'
+	docker compose down
 
 connectdb: ## proxies the db in kubernetes via kubectl
 	@echo "run psql -U forge -h 127.0.0.1 forge to connect"

apps/core/lib/core/pubsub/events.ex

@@ -94,3 +94,4 @@ defmodule Core.PubSub.UpgradesPromoted, do: use Piazza.PubSub.Event
 defmodule Core.PubSub.ConsoleInstanceCreated, do: use Piazza.PubSub.Event
 defmodule Core.PubSub.ConsoleInstanceUpdated, do: use Piazza.PubSub.Event
 defmodule Core.PubSub.ConsoleInstanceDeleted, do: use Piazza.PubSub.Event
+defmodule Core.PubSub.ConsoleInstanceReaped, do: use Piazza.PubSub.Event

apps/core/lib/core/schema/console_instance.ex

@@ -26,7 +26,9 @@ defmodule Core.Schema.ConsoleInstance do
     field :size,          Size
     field :region,        :string
 
-    field :deleted_at,    :utc_datetime_usec
+    field :first_notif_at,  :utc_datetime_usec
+    field :second_notif_at, :utc_datetime_usec
+    field :deleted_at,      :utc_datetime_usec
 
     embeds_one :instance_status, InstanceStatus, on_replace: :update do
       field :db,  :boolean, default: false
@@ -66,6 +68,23 @@ defmodule Core.Schema.ConsoleInstance do
     )
   end
 
+  def unpaid(query \\ __MODULE__) do
+    from(c in query,
+      join: u in assoc(c, :owner),
+      join: a in assoc(u, :account),
+      left_join: s in assoc(a, :subscription),
+      where: not is_nil(a.delinquent_at) or is_nil(s.id)
+    )
+  end
+
+  def reapable(query \\ __MODULE__) do
+    week_ago = Timex.now() |> Timex.shift(weeks: -1)
+    default = Timex.shift(week_ago, weeks: -1)
+    from(c in query,
+      where: coalesce(coalesce(c.second_notif_at, c.first_notif_at), ^default) < ^week_ago
+    )
+  end
+
   def ordered(query \\ __MODULE__, order \\ [asc: :name]) do
     from(c in query, order_by: ^order)
   end

apps/core/lib/core/services/cloud.ex

@@ -98,6 +98,25 @@ defmodule Core.Services.Cloud do
     |> notify(:delete, user)
   end
 
+  @doc """
+  Proceeds to attempt to reap a cloud cluster, we'll give two notifications, then
+  """
+  @spec reap(ConsoleInstance.t) :: console_resp
+  def reap(%ConsoleInstance{first_notif_at: nil} = inst),
+    do: notify_reaping(inst, :first_notif_at)
+  def reap(%ConsoleInstance{second_notif_at: nil} = inst),
+    do: notify_reaping(inst, :second_notif_at)
+  def reap(%ConsoleInstance{} = inst) do
+    %{owner: owner} = Repo.preload(inst, [:owner])
+    delete_instance(inst.id, owner)
+  end
+
+  defp notify_reaping(instance, field) do
+    Ecto.Changeset.change(instance, %{field => Timex.now()})
+    |> Repo.update()
+    |> notify(:reap)
+  end
+
   def authorize(id, %User{} = user) do
     inst = get_instance!(id) |> Repo.preload([:owner])
     with {:ok, _} <- Core.Policies.Account.allow(inst.owner, user, :impersonate),
@@ -193,4 +212,8 @@ defmodule Core.Services.Cloud do
   defp notify({:ok, %ConsoleInstance{} = inst}, :delete, user),
     do: handle_notify(PubSub.ConsoleInstanceDeleted, inst, actor: user)
   defp notify(pass, _, _), do: pass
+
+  defp notify({:ok, %ConsoleInstance{} = inst}, :reap),
+    do: handle_notify(PubSub.ConsoleInstanceReaped, inst)
+  defp notify(pass, _), do: pass
 end

apps/core/priv/repo/migrations/20240806022151_add_cloud_schemas.exs

@@ -39,6 +39,9 @@ defmodule Core.Repo.Migrations.AddCloudSchemas do
       add :configuration, :map
       add :deleted_at,    :utc_datetime_usec
 
+      add :first_notif_at,  :utc_datetime_usec
+      add :second_notif_at, :utc_datetime_usec
+
       add :instance_status,        :map
 
       add :cockroach_id, references(:cockroach_clusters, type: :uuid)

apps/core/test/pubsub/fanout/cloud_test.ex → apps/core/test/pubsub/fanout/cloud_test.exs

@@ -1,4 +1,4 @@
-defmodule Core.PubSub.Fanout.ChartsTest do
+defmodule Core.PubSub.Fanout.CloudTest do
   use Core.SchemaCase, async: false
   alias Core.PubSub
   use Mimic
@@ -8,7 +8,7 @@ defmodule Core.PubSub.Fanout.ChartsTest do
       expect(Core.Conduit.Broker, :publish, fn msg, :cloud -> {:ok, msg} end)
 
       event = %PubSub.ConsoleInstanceCreated{item: insert(:console_instance)}
-      {:ok, ^event} = PubSub.Fanout.handle_event(event)
+      {:ok, %Conduit.Message{body: ^event}} = PubSub.Fanout.fanout(event)
     end
   end
 
@@ -17,7 +17,7 @@ defmodule Core.PubSub.Fanout.ChartsTest do
       expect(Core.Conduit.Broker, :publish, fn msg, :cloud -> {:ok, msg} end)
 
       event = %PubSub.ConsoleInstanceUpdated{item: insert(:console_instance)}
-      {:ok, ^event} = PubSub.Fanout.handle_event(event)
+      {:ok, %Conduit.Message{body: ^event}} = PubSub.Fanout.fanout(event)
     end
   end
 
@@ -26,7 +26,7 @@ defmodule Core.PubSub.Fanout.ChartsTest do
       expect(Core.Conduit.Broker, :publish, fn msg, :cloud -> {:ok, msg} end)
 
       event = %PubSub.ConsoleInstanceDeleted{item: insert(:console_instance)}
-      {:ok, ^event} = PubSub.Fanout.handle_event(event)
+      {:ok, %Conduit.Message{body: ^event}} = PubSub.Fanout.fanout(event)
     end
   end
 end

apps/core/test/services/cloud_test.exs

@@ -88,4 +88,39 @@ defmodule Core.Services.CloudTest do
       {:error, _} = Cloud.delete_instance(instance.id, user)
     end
   end
+
+  describe "#reap/1" do
+    test "it will send a first warning" do
+      inst = insert(:console_instance)
+
+      {:ok, reaped} = Cloud.reap(inst)
+
+      assert reaped.first_notif_at
+
+      assert_receive {:event, %PubSub.ConsoleInstanceReaped{item: ^reaped}}
+    end
+
+    test "it will send a second warning" do
+      inst = insert(:console_instance, first_notif_at: Timex.now())
+
+      {:ok, reaped} = Cloud.reap(inst)
+
+      assert reaped.second_notif_at
+
+      assert_receive {:event, %PubSub.ConsoleInstanceReaped{item: ^reaped}}
+    end
+
+    test "it will finally delete" do
+      inst = insert(:console_instance,
+        first_notif_at: Timex.now(),
+        second_notif_at: Timex.now()
+      )
+
+      {:ok, reaped} = Cloud.reap(inst)
+
+      assert reaped.deleted_at
+
+      assert_receive {:event, %PubSub.ConsoleInstanceDeleted{item: ^reaped}}
+    end
+  end
 end

apps/core/test/support/factory.ex

@@ -651,6 +651,7 @@ defmodule Core.Factory do
       size:        :small,
       cluster:     build(:cloud_cluster),
       cockroach:   build(:cockroach_cluster),
+      owner:       build(:user, service_account: true),
       subdomain:   "#{name}.cloud.plural.sh",
       url:         "console.#{name}.cloud.plural.sh",
       instance_status: %{db: true, svc: true},

apps/cron/lib/cron/prune/cloud.ex

@@ -0,0 +1,17 @@
+defmodule Cron.Prune.Cloud do
+  @moduledoc """
+  Reaps unpaid cloud consoles
+  """
+  use Cron
+  alias Core.Schema.{ConsoleInstance}
+  alias Core.Services.Cloud
+
+  def run() do
+    ConsoleInstance.unpaid()
+    |> ConsoleInstance.reapable()
+    |> ConsoleInstance.ordered(asc: :id)
+    |> Core.Repo.stream(method: :keyset)
+    |> Core.throttle()
+    |> Enum.each(&Cloud.reap/1)
+  end
+end

apps/cron/test/cron/prune/cloud_test.exs

@@ -0,0 +1,27 @@
+defmodule Cron.Prune.CloudTest do
+  use Core.SchemaCase
+  alias Cron.Prune.Cloud
+
+  describe "#run/0" do
+    test "it will prune old, unused invites" do
+      old = Timex.now() |> Timex.shift(weeks: -1) |> Timex.shift(minutes: -30)
+      account = insert(:account)
+      user    = insert(:user, account: account)
+      insert(:platform_subscription, account: account)
+
+      first   = insert(:console_instance)
+      ignore  = insert(:console_instance, owner: user)
+      ignore2 = insert(:console_instance, first_notif_at: Timex.now())
+      second  = insert(:console_instance, first_notif_at: old)
+      third   = insert(:console_instance, first_notif_at: old, second_notif_at: old)
+
+      Cloud.run()
+
+      assert refetch(first).first_notif_at
+      refute refetch(ignore).first_notif_at
+      refute refetch(ignore2).second_notif_at
+      assert refetch(second).second_notif_at
+      assert refetch(third).deleted_at
+    end
+  end
+end

apps/email/lib/email/builder/console_reaped.ex

@@ -0,0 +1,18 @@
+defmodule Email.Builder.ConsoleReaped do
+  use Email.Builder.Base
+  alias Core.Schema.ConsoleInstance
+
+  def email(inst) do
+    %{owner: user} = inst = Core.Repo.preload(inst, [:owner])
+
+    base_email()
+    |> to(expand_service_account(user))
+    |> subject("Your Plural Cloud Instance #{inst.name} is eligible to be decommissioned")
+    |> assign(:inst, inst)
+    |> assign(:warning, warning(inst))
+    |> render(:console_reaped)
+  end
+
+  defp warning(%ConsoleInstance{second_notif_at: nil}), do: 1
+  defp warning(_), do: 2
+end

apps/email/lib/email/deliverable/cloud.ex

@@ -0,0 +1,3 @@
+defimpl Email.Deliverable, for: Core.PubSub.ConsoleInstanceReaped do
+  def email(%{item: inst}), do: Email.Builder.ConsoleReaped.email(inst)
+end

apps/email/lib/email_web/templates/email/console_reaped.html.eex

@@ -0,0 +1,22 @@
+<div class="fit-content margin-horizontal-auto text-lead">
+  Your Plural Cloud instance <%= @inst.name %> is eligible to be deprovisioned.
+</div>
+<div class="fit-content margin-horizontal-auto text-content text-align-center margin-top-large">
+  You must have an active, paid plan to continue using Plural Cloud, please update your billing information and/or initiate a new subscription
+  to continue using your instance
+</div>
+
+<%= if @warning == 2 do %>
+<div class="fit-content margin-horizontal-auto text-content text-align-center margin-top-large">
+  Since your instance has been unpaid for 1 week, you have <b>only one more week</b> before we reap your instance
+</div>
+<% else %>
+<div class="fit-content margin-horizontal-auto text-content text-align-center margin-top-large">
+  We provide a <b>two week grace period</b> for delinquent accounts to fix any billing issues.  After that period, your instance will
+  be scheduled to be reaped.
+</div>
+<% end %>
+
+<a class="button margin-top-large" href="<%= url("/account/billing") %>">
+  Go to Billing
+</a>

apps/email/lib/email_web/templates/email/console_reaped.text.eex

@@ -0,0 +1,11 @@
+Your Plural Cloud instance is eligible to be deprovisioned.
+
+You must have an active, paid plan to continue using Plural Cloud, please update your billing information and/or initiate a new subscription to continue using your instance.
+
+<%= if @warning == 2 do %>
+Since your instance has been unpaid for 1 week, you have *only one more week* before we reap your instance
+<% else %>
+We provide a *two week grace period* for delinquent accounts to fix any billing issues.  After that period, your instance will be scheduled to be reaped.
+<% end %>
+
+Go to Billing here: <%= url("/account/billing") %>

apps/email/test/email/deliverable/cloud_test.exs

@@ -0,0 +1,37 @@
+defmodule Email.Deliverable.CloudTest do
+  use Core.SchemaCase, async: true
+  use Bamboo.Test
+
+  alias Core.PubSub
+  alias Email.PubSub.Consumer
+
+  describe "ConsoleInstanceReaped" do
+    test "it can send a first warning" do
+      owner = insert(:user, service_account: true)
+      insert(:impersonation_policy_binding,
+        policy: build(:impersonation_policy, user: owner),
+        user: build(:user)
+      )
+      inst = insert(:console_instance, owner: owner, first_notif_at: Timex.now())
+
+      event = %PubSub.ConsoleInstanceReaped{item: inst}
+      Consumer.handle_event(event)
+
+      assert_delivered_email Email.Builder.ConsoleReaped.email(inst)
+    end
+
+    test "it can send a second warning email" do
+      owner = insert(:user, service_account: true)
+      insert(:impersonation_policy_binding,
+        policy: build(:impersonation_policy, user: owner),
+        user: build(:user)
+      )
+      inst = insert(:console_instance, owner: owner, second_notif_at: Timex.now())
+
+      event = %PubSub.ConsoleInstanceReaped{item: inst}
+      Consumer.handle_event(event)
+
+      assert_delivered_email Email.Builder.ConsoleReaped.email(inst)
+    end
+  end
+end