Merge pull request #4840 from tptignor/actions-testing

github actions testing

.github/workflows/unit-test.yml

@@ -14,22 +14,82 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '11'
           distribution: 'adopt'
 
-      - name: Run Sbt Tests
+      - name: Run hmda-platform Tests
         run: |
-          touch log-file
-          sbt test > log-file
+          sbt "project hmda-platform" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run check-digit Tests
+        run: |
+          sbt "project check-digit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run common Tests
+        run: |
+          sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run data-browser Tests
+        run: |
+          sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run hmda-analytics Tests
+        run: |
+          sbt "project hmda-analytics" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run hmda-data-publisher Tests
+        run: |
+          sbt "project hmda-data-publisher" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run hmda-quarterly-data-service Tests
+        run: |
+          sbt "project hmda-quarterly-data-service" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run hmda-reporting Tests
+        run: |
+          sbt "project hmda-reporting" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run institutions-api Tests
+        run: |
+          sbt "project institutions-api" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run modified-lar Tests
+        run: |
+          sbt "project modified-lar" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run rate-limit Tests
+        run: |
+          sbt "project rate-limit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run ratespread-calculator Tests
+        run: |
+          sbt "project ratespread-calculator" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Run submission-errors Tests
+        run: |
+          sbt "project submission-errors" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
         continue-on-error: true
 
       - name: Check Test Results
         run: |
-          if [ $(cat log-file | grep -E "TEST FAILED|TESTS FAILED|Failed tests" | wc -l) -gt 0 ]; then
-            echo "Unit tests failed."
+          if [ $(grep 'All tests passed.' log-file | wc -l) -ne 13 ]; then
+            echo "One or more projects had failures. Please review the logs."
             exit 1
           else
-            echo "Unit tests passed."
+            echo "All tests passed."
+            exit 0
           fi

build.sbt

@@ -3,7 +3,7 @@ import BuildSettings._
 import sbtassembly.AssemblyPlugin.autoImport.assemblyMergeStrategy
 import com.typesafe.sbt.packager.docker._
 
-lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, embeddedPg, embeddedPgSupport, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi)
+lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, testContainers, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi)
 
 lazy val sparkDeps =
   Seq(

common/src/main/resources/reference.conf

@@ -314,7 +314,7 @@ db {
     url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig}
     url = ${?JDBC_URL}
     numThreads = 4
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

common/src/main/scala/hmda/util/CSVConsolidator.scala

@@ -10,11 +10,11 @@ object CSVConsolidator {
     seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.mkString(",")
   }
   def listDeDupeToList(seqToDeDupe: Seq[String]) = {
-    seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.toList
+    seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList
   }
 
    def stringDeDupeToList(stringToDeDupe: String) = {
-    stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.toList
+    stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList
   }
 
    def stringDeDupeToString(stringToDeDupe: String) = {

common/src/test/scala/hmda/utils/EmbeddedPostgres.scala

@@ -1,7 +1,7 @@
 package hmda.utils
 
 import org.scalatest.{ BeforeAndAfterAll, BeforeAndAfterEach, Suite }
-import ru.yandex.qatools.embed.postgresql.distribution.Version
+import org.testcontainers.containers.FixedHostPortGenericContainer
 import slick.basic.DatabaseConfig
 import slick.dbio.DBIO
 import slick.jdbc.JdbcProfile
@@ -16,7 +16,7 @@ import scala.concurrent.ExecutionContext.Implicits.global
  * after each test and finally deletes all tables before the suite is complete and tears down the database
  */
 trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: Suite =>
-  private val embeddedPg = new ru.yandex.qatools.embed.postgresql.EmbeddedPostgres(Version.V11_1)
+  private val testContainer = new FixedHostPortGenericContainer("postgres:12")
   val dbHoconpath        = "embedded-pg"
   val dbConfig           = DatabaseConfig.forConfig[JdbcProfile](dbHoconpath)
 
@@ -48,15 +48,19 @@ trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self:
   }
 
   override protected def beforeAll(): Unit = {
-    embeddedPg.start("localhost", 5432, "postgres", "postgres", "postgres")
+    testContainer.withEnv("POSTGRES_USER", "postgres")
+    testContainer.withEnv("POSTGRES_PASSWORD", "postgres")
+    testContainer.withEnv("POSTGRES_DB", "postgres")
+    testContainer.withFixedExposedPort(5432, 5432)
+    testContainer.start()
     executeSQL(removeAllTables)
     loadSqlFileFromResources(bootstrapSqlFile)
     super.beforeAll()
   }
 
   override protected def afterAll(): Unit = {
     executeSQL(removeAllTables)
-    embeddedPg.close()
+    testContainer.stop()
     super.afterAll()
   }
 

data-browser/src/test/resources/application.conf

@@ -17,7 +17,7 @@ embedded-pg {
     profile = slick.jdbc.PostgresProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala

@@ -15,6 +15,9 @@ import monix.execution.schedulers.TestScheduler
 import org.scalamock.scalatest.MockFactory
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.{ Assertion, Matchers, WordSpec }
+import org.scalatest.Tag
+
+object CustomTag extends Tag("actions-ignore")
 
 class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Matchers with MockFactory with ScalaFutures {
   "DataBrowserDirectives" must {
@@ -177,7 +180,7 @@ class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Ma
       }
     }
 
-    "extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" in {
+    "extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" taggedAs CustomTag in {
       val route: Route = failingRoute(extractMsaAndStateAndCountyAndInstitutionIdentifierBrowserFields)
 
       Get("/?msamds=34980&leis=BANK0&states=CA,AK&actions_taken=1,2,3&counties=19125&years=2018") ~> route ~> check {

hmda-analytics/src/test/resources/application.conf

@@ -22,7 +22,7 @@ embedded-pg {
     profile = slick.jdbc.PostgresProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

hmda-dashboard/src/main/resources/application.conf

@@ -45,7 +45,7 @@ dashboard_db {
     url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig}
     url = ${?JDBC_URL}
     numThreads = 10
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 2000000
     validationTimeout = 1000000
   }

hmda-data-publisher/src/test/resources/application.conf

@@ -143,7 +143,7 @@ embedded-pg {
     profile = slick.jdbc.PostgresProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

hmda-reporting/src/test/resources/application.conf

@@ -24,7 +24,7 @@ embedded-pg {
     profile = slick.jdbc.PostgresProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala

@@ -12,6 +12,9 @@ import hmda.persistence.AkkaCassandraPersistenceSpec
 import akka.actor.typed.scaladsl.adapter._
 import hmda.messages.institution.InstitutionCommands._
 import hmda.messages.institution.InstitutionEvents._
+import org.scalatest.Tag
+
+object CustomTag extends Tag("actions-ignore")
 
 class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec {
 
@@ -44,7 +47,7 @@ class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec {
       maybeInstitutionProbe.expectMessage(Some(sampleInstitution))
     }
 
-    "not be created if it already exists" in {
+    "not be created if it already exists" taggedAs CustomTag in {
       val institutionPersistence =
         system.spawn(InstitutionPersistence.behavior("ABC12345"), actorName)
       institutionPersistence ! CreateInstitution(sampleInstitution, institutionProbe.ref)

modified-lar/src/test/resources/application.conf

@@ -76,7 +76,7 @@ embedded-pg {
     profile = slick.jdbc.PostgresProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }

project/BuildSettings.scala

@@ -21,7 +21,7 @@ object BuildSettings {
       scalaVersion := buildScalaVersion,
       scalacOptions ++= Seq("-Xlint", "-deprecation", "-unchecked", "-feature", "-Ymacro-annotations"),
       assembly / aggregate := false,
-      Test / parallelExecution:= true,
+      Test / parallelExecution:= false,
       Test / fork := true,
       resolvers += "Akka library repository".at("https://repo.akka.io/maven"),
       commands += dockerPublishLocalSkipTestsCommand

project/Dependencies.scala

@@ -54,9 +54,7 @@ object Dependencies {
   lazy val alpakkaSlick             = "com.lightbend.akka" %% "akka-stream-alpakka-slick"           % Version.alpakka
   lazy val postgres                 = "org.postgresql"     % "postgresql"                           % Version.postgres
   lazy val h2                       = "com.h2database"     % "h2"                                   % Version.h2 % Test
-  lazy val embeddedPg =
-    "ru.yandex.qatools.embed" % "postgresql-embedded" % Version.embeddedPg % Test exclude ("de.flapdoodle.embed", "de.flapdoodle.embed.process")
-  lazy val embeddedPgSupport     = "de.flapdoodle.embed"        % "de.flapdoodle.embed.process" % "2.1.2" % Test
+  lazy val testContainers        = "org.testcontainers"         % "testcontainers"              % Version.testContainers % "test"
   lazy val s3Mock                = "com.adobe.testing"          % "s3mock"                      % "2.1.19" % Test
   lazy val apacheCommonsIO       = "commons-io"                 % "commons-io"                  % "2.6" % Test
   lazy val keycloakAdapter       = "org.keycloak"               % "keycloak-adapter-core"       % Version.keycloak

project/Version.scala

@@ -15,7 +15,6 @@ object Version {
   val slickPostgres          = "0.21.1"
   val postgres               = "42.6.0"
   val h2                     = "2.1.214"
-  val embeddedPg             = "2.10"
   val keycloak               = "22.0.1"
   val resteasy               = "6.2.4.Final"
   val jbossLogging           = "3.5.1.Final"
@@ -39,4 +38,5 @@ object Version {
   val diffx                  = "0.4.0"
   val log4j                  = "2.15.0"
   val kubernetesApi          = "15.0.1"
+  val testContainers         = "1.19.8"
 }

submission-errors/src/test/resources/application.conf

@@ -15,7 +15,7 @@ embedded-pg {
     profile = hmda.submissionerrors.repositories.PostgresEnhancedProfile
     url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
     numThreads = 2
-    queueSize = 10
+    queueSize = 100
     connectionTimeout = 20000
     validationTimeout = 10000
   }