diff --git a/apps/core/lib/core/retry.ex b/apps/core/lib/core/retry.ex new file mode 100644 index 000000000..96620ac28 --- /dev/null +++ b/apps/core/lib/core/retry.ex @@ -0,0 +1,20 @@ +defmodule Core.Retry do + defstruct [wait: 200, attempts: 0, max: 3] + + def retry(fun, args) when is_function(fun) and (is_list(args) or is_map(args)) do + struct(__MODULE__, Map.new(args)) + |> retry(fun) + end + + def retry(%__MODULE__{attempts: attempts, max: max, wait: wait} = conf, fun) do + case {attempts < max, fun.()} do + {_, :ok} -> :ok + {_, {:ok, res}} -> {:ok, res} + {true, {:error, _} = error} -> + Logger.info "failed to execute function, error: #{inspect(error)}" + :timer.sleep(wait) + retry(%{conf | attempts: attempts + 1}, fun) + {_, {:error, err}} -> {:error, err} + end + end +end diff --git a/apps/core/lib/core/services/cloud/workflow/shared.ex b/apps/core/lib/core/services/cloud/workflow/shared.ex index f9d804b26..8c2afc7d8 100644 --- a/apps/core/lib/core/services/cloud/workflow/shared.ex +++ b/apps/core/lib/core/services/cloud/workflow/shared.ex @@ -20,12 +20,13 @@ defmodule Core.Services.Cloud.Workflow.Shared do def sync(_), do: :ok def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do - :timer.sleep(:timer.seconds(5)) - case {DNS.resolve(url), DNS.resolve(url, :cname)} do - {{:ok, [_ | _]}, _} -> mark_provisioned(inst) - {_, {:ok, [_ | _]}} -> mark_provisioned(inst) - {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} - end + Core.Retry.retry(fn -> + case {DNS.resolve(url), DNS.resolve(url, :cname)} do + {{:ok, [_ | _]}, _} -> mark_provisioned(inst) + {_, {:ok, [_ | _]}} -> mark_provisioned(inst) + {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} + end + end, wait: :timer.seconds(30), max: 100) end def up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do