Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix snake CVE #4837

Merged
merged 16 commits into from
Aug 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import BuildSettings._
import sbtassembly.AssemblyPlugin.autoImport.assemblyMergeStrategy
import com.typesafe.sbt.packager.docker._

lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, testContainers, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi)
lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, testContainers, apacheCommonsIO, log4jToSlf4j, kubernetesApi)

lazy val sparkDeps =
Seq(
Expand Down Expand Up @@ -50,7 +50,7 @@ lazy val akkaPersistenceDeps =
)

lazy val akkaHttpDeps =
Seq(akkaHttp, akkaHttp2, akkaHttpTestkit, akkaStreamsTestKit, akkaHttpCirce, akkaHttpXml)
Seq(akkaHttp, akkaHttp2, akkaHttpXml, akkaHttpTestkit, akkaStreamsTestKit, akkaHttpCirce)
lazy val circeDeps = Seq(circe, circeGeneric, circeParser)
lazy val enumeratumDeps = Seq(enumeratum, enumeratumCirce)

Expand Down Expand Up @@ -162,6 +162,7 @@ lazy val `hmda-platform` = (project in file("hmda"))
case "logback.xml" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixes merge strategy errors, added for all services

case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -197,10 +198,12 @@ lazy val `check-digit` = (project in file("check-digit"))
s"${name.value}.jar"
},
assembly / assemblyMergeStrategy := {

case "application.conf" => MergeStrategy.concat
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs @ _*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs @ _*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -241,6 +244,7 @@ lazy val `check-digit` = (project in file("check-digit"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -276,6 +280,7 @@ lazy val `institutions-api` = (project in file("institutions-api"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -318,6 +323,7 @@ lazy val `hmda-data-publisher` = (project in file("hmda-data-publisher"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -355,6 +361,7 @@ lazy val `hmda-dashboard` = (project in file("hmda-dashboard"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -393,6 +400,7 @@ lazy val `ratespread-calculator` = (project in file("ratespread-calculator"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -431,6 +439,7 @@ lazy val `modified-lar` = (project in file("modified-lar"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -470,6 +479,7 @@ lazy val `irs-publisher` = (project in file("irs-publisher"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -509,6 +519,7 @@ lazy val `hmda-reporting` = (project in file("hmda-reporting"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -557,6 +568,7 @@ lazy val `hmda-analytics` = (project in file("hmda-analytics"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -598,6 +610,7 @@ lazy val `hmda-analytics` = (project in file("hmda-analytics"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs @ _*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs @ _*) => MergeStrategy.last
case "reference.conf" => MergeStrategy.concat
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
Expand Down Expand Up @@ -635,6 +648,7 @@ lazy val `rate-limit` = (project in file("rate-limit"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -674,6 +688,7 @@ lazy val `data-browser` = (project in file("data-browser"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -707,6 +722,7 @@ lazy val `submission-errors` = (project in file("submission-errors"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps@_*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -737,6 +753,7 @@ lazy val `email-service` = (project in file("email-service"))
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down Expand Up @@ -777,6 +794,7 @@ lazy val `hmda-quarterly-data-service` = (project in file ("hmda-quarterly-data-
case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
case "META-INF/MANIFEST.MF" => MergeStrategy.discard
case PathList("META-INF", xs@_*) => MergeStrategy.concat
case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first
case PathList("jakarta", xs@_*) => MergeStrategy.last
case PathList(ps @ _*) if ps.last endsWith ".proto" =>
MergeStrategy.first
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/resources/persistence.conf
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ datastax-java-driver {
basic {
contact-points = ["localhost:9042"]
contact-points = [${?CASSANDRA_CLUSTER_HOSTS}":9042"]
load-balancing-policy.local-datacenter = ""
load-balancing-policy.local-datacenter = "datacenter1"
load-balancing-policy.local-datacenter = ${?CASSANDRA_CLUSTER_DC}
}
advanced {
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/scala/hmda/util/RealTimeConfig.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class RealTimeConfig(val cmName: String, val ns: String) {
val factory = new SharedInformerFactory(client)
val informer = factory.sharedIndexInformerFor((params: CallGeneratorParams) => {
api.listNamespacedConfigMapCall(
ns, null, null, null, s"metadata.name=$cmName", null, null, params.resourceVersion, null, params.timeoutSeconds, params.watch, null)
ns, null, null, null, s"metadata.name=$cmName", null, null, params.resourceVersion, null, null, params.timeoutSeconds, params.watch, null)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Due to the kubernetes client-java dependency upgrade, this method now has another parameter, so I added a null parameter value here

}, classOf[V1ConfigMap], classOf[V1ConfigMapList])
informer.addEventHandler(new ResourceEventHandler[V1ConfigMap] {
override def onAdd(obj: V1ConfigMap): Unit = {
Expand Down
33 changes: 0 additions & 33 deletions common/src/test/scala/hmda/utils/EmbeddedS3.scala
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deleted since it was not used and relied on S3Mock dependency, which I removed to avoid some big dependency version issues

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import io.lettuce.core.{ ClientOptions, RedisClient }
import monix.eval.Task
import slick.basic.DatabaseConfig
import slick.jdbc.JdbcProfile
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.cors
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All the cors related changes in these files fix the "ambiguous reference errors" from the new akka dependency version.


import scala.concurrent.ExecutionContext
import scala.concurrent.duration._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package hmda.publisher.api
import akka.http.scaladsl.model.headers.RawHeader
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.publisher.scheduler.AllSchedulers
import hmda.publisher.scheduler.schedules.{ Schedule, ScheduleWithYear, Schedules }
Expand Down
9 changes: 5 additions & 4 deletions hmda/src/main/scala/hmda/api/http/HmdaPublicApi.scala
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package hmda.api.http

import akka.actor.typed.{ ActorSystem, Behavior }
import akka.actor.{ CoordinatedShutdown, ActorSystem => ClassicActorSystem }
import akka.actor.typed.{ActorSystem, Behavior}
import akka.actor.{CoordinatedShutdown, ActorSystem => ClassicActorSystem}
import akka.actor.typed.scaladsl.Behaviors
import akka.stream.Materializer
import hmda.api.http.public.{ HmdaFileValidationHttpApi, LarValidationHttpApi, TsValidationHttpApi, HmdaFileParsingHttpApi }
import hmda.api.http.public.{HmdaFileParsingHttpApi, HmdaFileValidationHttpApi, LarValidationHttpApi, TsValidationHttpApi}
import hmda.api.http.routes.BaseHttpApi
import hmda.api.http.directives.HmdaTimeDirectives._

import scala.concurrent.ExecutionContext
import akka.http.scaladsl.server.Directives._
import akka.actor.typed.scaladsl.adapter._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.cors

// This is just a Guardian for starting up the API
// $COVERAGE-OFF$
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import akka.http.scaladsl.model.{ StatusCodes, Uri }
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import com.typesafe.config.Config
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import akka.cluster.sharding.typed.scaladsl.ClusterSharding
import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import com.typesafe.config.Config
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.auth.OAuth2Authorization
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import akka.http.scaladsl.model.{StatusCodes, Uri}
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.directives.CreateFilingAuthorization._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import akka.http.scaladsl.model.Uri
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.directives.QuarterlyFilingAuthorization.quarterlyFilingAllowed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.{StatusCodes, Uri}
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.directives.QuarterlyFilingAuthorization._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import akka.stream.Materializer
import akka.stream.scaladsl.Sink
import akka.util.{ ByteString, Timeout }
import com.typesafe.config.Config
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.directives.CreateFilingAuthorization._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import akka.http.scaladsl.model.{ StatusCodes, Uri }
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.directives.QuarterlyFilingAuthorization._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import akka.http.scaladsl.server.Route
import akka.stream.scaladsl.{ Broadcast, Concat, Flow, GraphDSL, Sink, Source }
import akka.stream.{ FlowShape, Materializer }
import akka.util.ByteString
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.model.filing.submissions.HmdaRowParsedErrorSummary
import hmda.api.http.utils.ParserErrorUtils
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@ import akka.http.scaladsl.server.Route
import akka.stream.scaladsl.{ Broadcast, Concat, Flow, GraphDSL, Sink, Source }
import akka.stream.{ FlowShape, Materializer }
import akka.util.ByteString
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.model.validation.ValidationError
import hmda.model.validation.LarValidationError
import hmda.api.http.model.filing.submissions.HmdaRowParsedErrorSummary
import hmda.api.http.model.filing.submissions.{ ValidationErrorSummary, SingleValidationErrorSummary }
import hmda.api.http.utils.ParserErrorUtils
import hmda.model.validation.LarValidationError
import hmda.model.validation.TsValidationError
import hmda.validation.engine._
import hmda.validation.context.ValidationContext
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import akka.http.scaladsl.marshalling.ToResponseMarshallable
import akka.http.scaladsl.model.headers.RawHeader
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.model.public.LarValidateRequest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import akka.http.scaladsl.marshalling.ToResponseMarshallable
import akka.http.scaladsl.model.headers.RawHeader
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import ch.megard.akka.http.cors.scaladsl.CorsDirectives._
import ch.megard.akka.http.cors.scaladsl.CorsDirectives.{cors, corsRejectionHandler}
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import hmda.api.http.PathMatchers._
import hmda.api.http.model.public.TsValidateRequest
Expand Down
Loading
Loading