Merge pull request #26 from fossapps/retry-policy

feat(Program): retry connection and graceful fail
fossapps · Nov 16, 2019 · 36e0fa1 · 36e0fa1
2 parents 3cfbaa4 + 863f197
commit 36e0fa1
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 4 deletions.
diff --git a/Micro.KeyStore.Api/Program.cs b/Micro.KeyStore.Api/Program.cs
@@ -1,29 +1,79 @@
 using System;
+using System.Threading.Tasks;
 using App.Metrics;
 using App.Metrics.AspNetCore;
 using App.Metrics.Extensions.Configuration;
 using App.Metrics.Formatters.InfluxDB;
 using Micro.KeyStore.Api.Models;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 
 namespace Micro.KeyStore.Api
 {
     public class Program
     {
-        public static void Main(string[] args)
+        public static async Task Main(string[] args)
         {
             var host = CreateHostBuilder(args).Build();
             using (var scope = host.Services.CreateScope())
             {
-                scope.ServiceProvider.GetRequiredService<ApplicationContext>().Database.Migrate();
+                var logger = scope.ServiceProvider.GetRequiredService<ILogger<Program>>();
+                try
+                {
+                    var db = scope.ServiceProvider.GetRequiredService<ApplicationContext>().Database;
+                    await MigrateOrFail(db, logger);
+                }
+                catch (RetryLimitExceededException e)
+                {
+                    logger.LogCritical(e, "Error connecting to database, application can't start");
+                    Environment.ExitCode = 1;
+                    return;
+                }
             }
             host.Run();
         }
 
+        private static async Task MigrateOrFail(DatabaseFacade db, ILogger logger)
+        {
+            for (var i = 0; i <= 3; i++)
+            {
+                var waitTime = new[] {1, 3, 8, 10}[i];
+                logger.LogInformation($"Db connection attempt in {waitTime} seconds");
+                await Task.Delay(TimeSpan.FromSeconds(waitTime));
+
+                if (await TryMigrate(db))
+                {
+                    return;
+                }
+                logger.LogWarning("Connection failed...");
+            }
+            throw new RetryLimitExceededException("Couldn't connect to database");
+        }
+
+        private static async Task<bool> TryMigrate(DatabaseFacade db)
+        {
+            try
+            {
+                var canConnect = await db.CanConnectAsync();
+                if (!canConnect)
+                {
+                    return false;
+                }
+                await db.MigrateAsync();
+                return true;
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
         public static IHostBuilder CreateHostBuilder(string[] args) =>
             Host.CreateDefaultBuilder(args)
                 .ConfigureWebHostDefaults(webBuilder =>

diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -2,8 +2,6 @@ version: '3'
 services:
   api:
     image: fossapps/micro.keystore
-    depends_on:
-      - postgres
     ports:
       - 5000:5000
     environment: