Merge pull request #4446 from PatrickGoRaft/data-broswer-2021

Data broswer 2021
cfpb · May 26, 2022 · 234c3c6 · 234c3c6
2 parents 9db1ea8 + bfe7b70
commit 234c3c6
Show file tree

Hide file tree

Showing 10 changed files with 1,115 additions and 14 deletions.
diff --git a/data-browser/src/main/resources/application.conf b/data-browser/src/main/resources/application.conf
@@ -29,9 +29,10 @@ server {
     routes {
       filtered-queries {
         2017 = "data-browser/2017/filtered-queries"
-        2018 = "data-browser/filtered-queries"
+        2018 = "data-browser/2018/filtered-queries"
         2019 = "data-browser/2019/filtered-queries"
         2020 = "data-browser/2020/filtered-queries"
+        2021 = "data-browser/2021/filtered-queries"
       }
       // this is where s3 files with unique md5 hash strings are saved
     }
@@ -50,13 +51,16 @@ dbconfig {
   database = ${?PG_DATABASE}
 
   table {
-    2020 = "modifiedlar2020_snapshot"
+    2021 = "modifiedlar2021_snapshot_04302022"
+    2021 = ${?DATABROWSER_PG_TABLE_2021}
+
+    2020 = "modifiedlar2020_one_year_04302022"
     2020 = ${?DATABROWSER_PG_TABLE_2020}
 
-    2019 = "modifiedlar2019_snapshot"
+    2019 = "modifiedlar2019_one_year_04052022"
     2019 = ${?DATABROWSER_PG_TABLE_2019}
 
-    2018 = "modifiedlar2018_snapshot"
+    2018 = "modifiedlar2018_three_year_04052022"
     2018 = ${?DATABROWSER_PG_TABLE_2018}
 
     2017 = "modifiedlar2017_ultimate"

diff --git a/data-browser/src/main/scala/hmda/dataBrowser/Settings.scala b/data-browser/src/main/scala/hmda/dataBrowser/Settings.scala
@@ -30,6 +30,7 @@ trait Settings {
   }
 
   object database {
+    val tableName2021: String = config.getString("dbconfig.table.2021")
     val tableName2020: String = config.getString("dbconfig.table.2020")
     val tableName2019: String = config.getString("dbconfig.table.2019")
     val tableName2018: String = config.getString("dbconfig.table.2018")
@@ -41,7 +42,8 @@ trait Settings {
         case 2018 => tableName2018
         case 2019 => tableName2019
         case 2020 => tableName2020
-        case _    => tableName2020
+        case 2021 => tableName2021
+        case _    => tableName2021
       }
       ModifiedLarTable(selected)
     }
@@ -74,7 +76,8 @@ trait Settings {
         case 2018 => config.getString("server.s3.routes.filtered-queries.2018")
         case 2019 => config.getString("server.s3.routes.filtered-queries.2019")
         case 2020 => config.getString("server.s3.routes.filtered-queries.2020")
-        case _ => config.getString("server.s3.routes.filtered-queries.2020")
+        case 2021 => config.getString("server.s3.routes.filtered-queries.2021")
+        case _ => config.getString("server.s3.routes.filtered-queries.2021")
       }
       selected
     }

diff --git a/data-browser/src/main/scala/hmda/dataBrowser/api/DataBrowserDirectives.scala b/data-browser/src/main/scala/hmda/dataBrowser/api/DataBrowserDirectives.scala
@@ -33,10 +33,12 @@ import hmda.dataBrowser.services._
 import monix.eval.Task
 import hmda.dataBrowser.Settings
 import enumeratum._
+import org.slf4j.LoggerFactory
 
 import scala.util.Try
 
 trait DataBrowserDirectives extends Settings {
+  val log = LoggerFactory.getLogger(getClass)
   private implicit val csvStreamingSupport: CsvEntityStreamingSupport =
     EntityStreamingSupport.csv()
 
@@ -77,14 +79,19 @@ trait DataBrowserDirectives extends Settings {
       .retrieveDataUrl(queries.queryFields, delimiter, year)
       .flatMap {
         case Some(url) =>
+          log.info("DataBrowserDirectives redirecting: {}", url)
           Task.now(Right(url))
         case None =>
           // upload the data to S3 in the background and emit the Source immediately
+          log.info("DataBrowserDirectives uploading")
           cache
             .persistData(queries.queryFields, delimiter, year, serializedData)
             .startAndForget *> Task(Left(serializedData))
       }
-      .onErrorFallbackTo(Task.now(Left(serializedData)))
+      .onErrorHandleWith { error =>
+        log.error("DataBrowserDirectives onErrorHandleWith Something failed", error)
+        Task.now(Left(serializedData))
+      }
   }
 
   def csvSource(s: Source[ModifiedLarEntity, NotUsed]): Source[ByteString, NotUsed] = {
@@ -166,7 +173,7 @@ trait DataBrowserDirectives extends Settings {
         import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
         complete((BadRequest, "must provide years value parameter"))
 
-      case xs if xs.exists(year => (year < 2018) || (year > 2020)) => // TODO: Change this to 2017 when 2017 is released
+      case xs if xs.exists(year => (year < 2018) || (year > 2021)) => // TODO: Change this to 2017 when 2017 is released
         import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
         complete((BadRequest, s"must provide years in the range of 2018-2020, you have provided (${xs.mkString(", ")})"))
 

diff --git a/...-browser/src/main/scala/hmda/dataBrowser/repositories/PostgresModifiedLarRepository.scala b/...-browser/src/main/scala/hmda/dataBrowser/repositories/PostgresModifiedLarRepository.scala
@@ -167,10 +167,11 @@ class PostgresModifiedLarRepository(config: DatabaseConfig[JdbcProfile], tableSe
 
   override def findFilers(filerFields: List[QueryField], year: Int): Task[Seq[FilerInformationLatest]] = {
     val institutionsTableName = year match { //will be needed when data browser has to support multiple years
-      case 2018 => "institutions2018_snapshot"
-      case 2019 => "institutions2019_snapshot"
-      case 2020 => "institutions2020_snapshot"
-      case _    => "institutions2020_snapshot"
+      case 2018 => "institutions2018_three_year_04052022"
+      case 2019 => "institutions2019_one_year_04052022"
+      case 2020 => "institutions2020_one_year_04302022"
+      case 2021 => "institutions2021_snapshot_04302022"
+      case _    => "institutions2021_snapshot_04302022"
     }
 
     //do not include year in the WHERE clause because all entries in the table (modifiedlar2018_snapshot) have filing_year = 2018

diff --git a/data-browser/src/main/scala/hmda/dataBrowser/services/S3FileService.scala b/data-browser/src/main/scala/hmda/dataBrowser/services/S3FileService.scala
@@ -43,9 +43,11 @@ class S3FileService(implicit mat: Materializer) extends FileService with Setting
     )
 
     Task.deferFuture {
+      log.info("S3FileService Starting upload")
       dataSource.runWith(sink)
     }.onErrorHandleWith { error =>
       // Note: (this *> that) comes from using cats.implicits and it means execute `this` and discard results then run `that`
+      log.error("S3FileService Something failed", error)
       Task.eval(log.error(s"Failed to write data to the S3 bucket (Extended info: ${error.toString})", error)) *> Task
         .raiseError(error)
     }.void

diff --git a/data-browser/src/test/resources/modifiedlar.sql b/data-browser/src/test/resources/modifiedlar.sql
@@ -1,4 +1,4 @@
-CREATE TABLE institutions2018_snapshot (
+CREATE TABLE modifiedlar2018_three_year_04052022 (
     lei character varying NOT NULL,
     activity_year integer NOT NULL,
     agency integer NOT NULL,
@@ -19,7 +19,7 @@ CREATE TABLE institutions2018_snapshot (
     quarterly_filer boolean default false NOT NULL
 );
 
-CREATE TABLE modifiedlar2018_snapshot (
+CREATE TABLE modifiedlar2019_one_year_04052022 (
     id integer NOT NULL,
     lei character varying NOT NULL,
     loan_type integer,