From 0e64f2ad55416036f45e1e8e548e0d48eacbd746 Mon Sep 17 00:00:00 2001
From: michaeljguarino <mjg@plural.sh>
Date: Tue, 24 Sep 2024 16:40:46 -0400
Subject: [PATCH] Better dns resolver retries

Not sure this will work, but if it doesn't we can just skip the dns test
---
 apps/core/lib/core/retry.ex                   | 20 +++++++++++++++++++
 .../core/services/cloud/workflow/shared.ex    | 13 ++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)
 create mode 100644 apps/core/lib/core/retry.ex

diff --git a/apps/core/lib/core/retry.ex b/apps/core/lib/core/retry.ex
new file mode 100644
index 000000000..96620ac28
--- /dev/null
+++ b/apps/core/lib/core/retry.ex
@@ -0,0 +1,20 @@
+defmodule Core.Retry do
+  defstruct [wait: 200, attempts: 0, max: 3]
+
+  def retry(fun, args) when is_function(fun) and (is_list(args) or is_map(args)) do
+    struct(__MODULE__, Map.new(args))
+    |> retry(fun)
+  end
+
+  def retry(%__MODULE__{attempts: attempts, max: max, wait: wait} = conf, fun) do
+    case {attempts < max, fun.()} do
+      {_, :ok} -> :ok
+      {_, {:ok, res}} -> {:ok, res}
+      {true, {:error, _} = error} ->
+        Logger.info "failed to execute function, error: #{inspect(error)}"
+        :timer.sleep(wait)
+        retry(%{conf | attempts: attempts + 1}, fun)
+      {_, {:error, err}} -> {:error, err}
+    end
+  end
+end
diff --git a/apps/core/lib/core/services/cloud/workflow/shared.ex b/apps/core/lib/core/services/cloud/workflow/shared.ex
index f9d804b26..8c2afc7d8 100644
--- a/apps/core/lib/core/services/cloud/workflow/shared.ex
+++ b/apps/core/lib/core/services/cloud/workflow/shared.ex
@@ -20,12 +20,13 @@ defmodule Core.Services.Cloud.Workflow.Shared do
   def sync(_), do: :ok
 
   def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do
-    :timer.sleep(:timer.seconds(5))
-    case {DNS.resolve(url), DNS.resolve(url, :cname)} do
-      {{:ok, [_ | _]}, _} -> mark_provisioned(inst)
-      {_, {:ok, [_ | _]}} -> mark_provisioned(inst)
-      {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"}
-    end
+    Core.Retry.retry(fn ->
+      case {DNS.resolve(url), DNS.resolve(url, :cname)} do
+        {{:ok, [_ | _]}, _} -> mark_provisioned(inst)
+        {_, {:ok, [_ | _]}} -> mark_provisioned(inst)
+        {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"}
+      end
+    end, wait: :timer.seconds(30), max: 100)
   end
 
   def up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do