Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3642 defer all credible set resolvers #245

Merged
merged 7 commits into from
Dec 3, 2024
Merged

3642 defer all credible set resolvers #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ea3356b
genericize the deferred resolver
jdhayhurst Nov 28, 2024
239fb11
formatting
jdhayhurst Nov 28, 2024
7a1232a
deferred resolver for credible sets by study using multi search api
jdhayhurst Nov 29, 2024
0461bc9
deferred resolver for credset by variant
jdhayhurst Nov 30, 2024
d423842
add ResultsHandler
jdhayhurst Dec 2, 2024
71f350d
refactor elastic retriever to handle multi search and nested responses
jdhayhurst Dec 2, 2024
a309a38
increase page size defaults for resolvers
jdhayhurst Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 112 additions & 46 deletions app/models/Backend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import models.entities.SearchFacetsResults._
import models.entities._
import models.gql.Arguments.variantId
import models.gql.StudyTypeEnum
import models.InnerResults
import models.Results
import org.apache.http.impl.nio.reactor.IOReactorConfig
import play.api.cache.AsyncCacheApi
import play.api.db.slick.DatabaseConfigProvider
Expand Down Expand Up @@ -103,10 +105,10 @@ class Backend @Inject() (implicit
ElasticRetriever.sortByDesc("llr")
)
.map {
case (Seq(), _, _) =>
case Results(Seq(), _, _, _) =>
logger.debug(s"No adverse event found for ${kv.toString}")
None
case (seq, agg, _) =>
case Results(seq, agg, _, _) =>
logger.trace(Json.prettyPrint(agg))
val counts = (agg \ "eventCount" \ "value").as[Long]
Some(AdverseEvents(counts, seq.head.criticalValue, seq))
Expand All @@ -126,8 +128,8 @@ class Backend @Inject() (implicit
)

esRetriever.getByIndexedQueryMust(cbIndex, kv, pag, fromJsValue[DiseaseHPO], aggs).map {
case (Seq(), _, _) => Some(DiseaseHPOs(0, Seq()))
case (seq, agg, _) =>
case Results(Seq(), _, _, _) => Some(DiseaseHPOs(0, Seq()))
case Results(seq, agg, _, _) =>
logger.trace(Json.prettyPrint(agg))
val rowsCount = (agg \ "rowsCount" \ "value").as[Long]
Some(DiseaseHPOs(rowsCount, seq))
Expand All @@ -146,23 +148,22 @@ class Backend @Inject() (implicit
.getByIndexedTermsMust(
indexName,
Map("studyLocusId.keyword" -> ids),
Pagination(Pagination.indexDefault, Pagination.sizeMax),
Pagination.mkMax,
fromJsValue[L2GPredictions],
sortByField = ElasticRetriever.sortBy("score", SortOrder.Desc)
)
.map(_._1)
.map(_.mappedHits)
}

def getVariants(ids: Seq[String]): Future[IndexedSeq[VariantIndex]] = {
val indexName = getIndexOrDefault("variant_index")
val pag = Pagination(Pagination.indexDefault, Pagination.sizeMax)
val r = esRetriever
.getByIndexedTermsMust(indexName,
Map("variantId.keyword" -> ids),
pag,
Pagination.mkMax,
fromJsValue[VariantIndex]
)
.map(_._1)
.map(_.mappedHits)
r
}

Expand All @@ -172,25 +173,24 @@ class Backend @Inject() (implicit
.getByIndexedTermsMust(
indexName,
Map("biosampleId.keyword" -> ids),
Pagination(Pagination.indexDefault, Pagination.sizeMax),
Pagination.mkMax,
fromJsValue[Biosample]
)
.map(_._1)
.map(_.mappedHits)
}

def getStudy(ids: Seq[String]): Future[IndexedSeq[JsValue]] = {
val pag = Pagination.mkDefault
val indexName = getIndexOrDefault("gwas_index")
val termsQuery = Map("studyId.keyword" -> ids)
val retriever =
esRetriever
.getByIndexedTermsMust(
indexName,
termsQuery,
pag,
Pagination.mkMax,
fromJsValue[JsValue]
)
retriever.map(_._1)
retriever.map(_.mappedHits)
}

def getStudies(queryArgs: StudyQueryArgs, pagination: Option[Pagination]): Future[Studies] = {
Expand Down Expand Up @@ -220,8 +220,8 @@ class Backend @Inject() (implicit
fromJsValue[JsValue]
)
retriever.map {
case (Seq(), _, _) => Studies.empty
case (studies, _, count) =>
case Results(Seq(), _, _, _) => Studies.empty
case Results(studies, _, count, _) =>
Studies(count, studies)
}
}
Expand All @@ -241,7 +241,7 @@ class Backend @Inject() (implicit
)
val colocs = esRetriever
.getByIndexedTermsShould(indexName, terms, pag, fromJsValue[Colocalisation], filter = filter)
.map(_._1)
.map(_.mappedHits)
colocs.map(_.map { coloc =>
val otherStudyLocusId: String = if (coloc.leftStudyLocusId != studyLocusId) {
coloc.leftStudyLocusId
Expand Down Expand Up @@ -288,33 +288,31 @@ class Backend @Inject() (implicit
.getInnerQ(
indexName,
query,
Pagination.mkDefault,
Pagination.mkMax,
fromJsValue[Locus],
"locus",
Some("studyLocusId")
)
retriever.map { case (locus, _, counts, studyLocusIds) =>
// zip the credsets with the counts and studyLocusIds and locusQueries and return an indexed seq of loci
retriever.map { case InnerResults(locus, _, counts, studyLocusIds) =>
locus.zip(counts).zip(studyLocusIds).map { case ((locus, count), studyLocusId) =>
Loci(count, Some(locus), studyLocusId.as[String])
}
}
}

def getCredibleSet(ids: Seq[String]): Future[IndexedSeq[JsValue]] = {
val pag = Pagination.mkDefault
val indexName = getIndexOrDefault("credible_set")
val termsQuery = Map("studyLocusId.keyword" -> ids)
val retriever =
esRetriever
.getByIndexedTermsMust(
indexName,
termsQuery,
pag,
Pagination.mkMax,
fromJsValue[JsValue],
excludedFields = Seq("locus", "ldSet")
)
retriever.map(_._1)
retriever.map(_.mappedHits)
}

def getCredibleSets(
Expand Down Expand Up @@ -349,7 +347,6 @@ class Backend @Inject() (implicit
must(termsQueryIter)
}
}
logger.info(s"Querying credible sets for: $query")
val retriever =
esRetriever
.getQ(
Expand All @@ -360,12 +357,89 @@ class Backend @Inject() (implicit
excludedFields = Seq("locus", "ldSet")
)
retriever.map {
case (Seq(), _, _) => CredibleSets.empty
case (credset, _, count) =>
case Results(Seq(), _, _, _) => CredibleSets.empty
case Results(credset, _, count, _) =>
CredibleSets(count, credset)
}
}

def getCredibleSetsByStudy(studyIds: Seq[String],
pagination: Option[Pagination]
): Future[IndexedSeq[CredibleSets]] = {
val pag = pagination.getOrElse(Pagination.mkDefault)
val indexName = getIndexOrDefault("credible_set")
val queries = studyIds.map { studyId =>
IndexQuery(
esIndex = indexName,
kv = Map("studyId.keyword" -> Seq(studyId)),
filters = Seq.empty,
pagination = pag,
aggs = Seq.empty,
excludedFields = Seq("locus", "ldSet")
)
}
val retriever =
esRetriever
.getMultiByIndexedTermsMust(
queries,
fromJsValue[JsValue],
None,
Some(ResolverField("studyId"))
)
retriever.map { case r =>
r.map {
case Results(Seq(), _, _, _) => CredibleSets.empty
case Results(credsets, _, counts, studyId) =>
CredibleSets(counts, credsets, studyId.as[String])
}
}
}

def getCredibleSetsByVariant(variantIds: Seq[String],
studyTypes: Option[Seq[StudyTypeEnum.Value]],
pagination: Option[Pagination]
): Future[IndexedSeq[CredibleSets]] = {
val pag = pagination.getOrElse(Pagination.mkDefault)
val indexName = getIndexOrDefault("credible_set")
val termsQueryIter: Option[Iterable[queries.Query]] = studyTypes match {
case Some(studyTypes) => Some(Iterable(should(termsQuery("studyType.keyword", studyTypes))))
case None => None
}
// nested query for each variant id in variantIds
val boolQueries: Seq[IndexBoolQuery] = variantIds.map { variantId =>
val query: BoolQuery = {
val nestedTermsQuery = termQuery("locus.variantId.keyword", variantId)
val nestedQueryIter =
Iterable(nestedQuery("locus", must(nestedTermsQuery)).inner(innerHits("locus").size(1)))
termsQueryIter match {
case None => must(nestedQueryIter)
case Some(termsQueryIter) => must(termsQueryIter ++ nestedQueryIter)
}
}.queryName(variantId)
IndexBoolQuery(
esIndex = indexName,
boolQuery = query,
pagination = pag,
excludedFields = Seq("ldSet", "locus")
)
}
val retriever =
esRetriever
.getMultiQ(
boolQueries,
fromJsValue[JsValue],
None,
Some(ResolverField(matched_queries = true))
)
retriever.map { case r =>
r.map {
case Results(Seq(), _, _, _) => CredibleSets.empty
case Results(credsets, _, counts, variantId) =>
CredibleSets(counts, credsets, variantId.as[String])
}
}
}

def getTargetEssentiality(ids: Seq[String]): Future[IndexedSeq[TargetEssentiality]] = {
val targetIndexName = getIndexOrDefault("target_essentiality")

Expand Down Expand Up @@ -557,7 +631,7 @@ class Backend @Inject() (implicit
Pagination(0, Pagination.sizeMax),
fromJsValue[MousePhenotype]
)
.map(_._1)
.map(_.mappedHits)
}

def getPharmacogenomicsByDrug(id: String): Future[IndexedSeq[Pharmacogenomics]] = {
Expand Down Expand Up @@ -587,7 +661,7 @@ class Backend @Inject() (implicit
Pagination(0, Pagination.sizeMax),
fromJsValue[Pharmacogenomics]
)
.map(_._1)
.map(_.mappedHits)
}

def getOtarProjects(ids: Seq[String]): Future[IndexedSeq[OtarProjects]] = {
Expand Down Expand Up @@ -620,22 +694,22 @@ class Backend @Inject() (implicit
val queryTerm = Map("id.keyword" -> ids)
esRetriever
.getByIndexedQueryShould(drugIndexName, queryTerm, Pagination(0, ids.size), fromJsValue[Drug])
.map(_._1)
.map(_.mappedHits)
}

def getMechanismsOfAction(id: String): Future[MechanismsOfAction] = {

logger.debug(s"querying ES: getting mechanisms of action for $id")
val index = getIndexOrDefault("drugMoA")
val queryTerms = Map("chemblIds.keyword" -> id)
val mechanismsOfActionRaw: Future[(IndexedSeq[MechanismOfActionRaw], JsValue, Long)] =
val mechanismsOfActionRaw: Future[Results[MechanismOfActionRaw]] =
esRetriever.getByIndexedQueryShould(
index,
queryTerms,
Pagination.mkDefault,
fromJsValue[MechanismOfActionRaw]
)
mechanismsOfActionRaw.map(i => Drug.mechanismOfActionRaw2MechanismOfAction(i._1))
mechanismsOfActionRaw.map(i => Drug.mechanismOfActionRaw2MechanismOfAction(i.mappedHits))
}

def getIndications(ids: Seq[String]): Future[IndexedSeq[Indications]] = {
Expand All @@ -645,7 +719,7 @@ class Backend @Inject() (implicit

esRetriever
.getByIndexedQueryShould(index, queryTerm, Pagination.mkDefault, fromJsValue[Indications])
.map(_._1)
.map(_.mappedHits)
}

def getDrugWarnings(id: String): Future[IndexedSeq[DrugWarning]] = {
Expand All @@ -660,7 +734,7 @@ class Backend @Inject() (implicit
This work around relates to ticket opentargets/platform#1506
*/
val drugWarnings =
results._1.foldLeft(Map.empty[(Option[Long]), DrugWarning]) { (dwMap, dw) =>
results.mappedHits.foldLeft(Map.empty[(Option[Long]), DrugWarning]) { (dwMap, dw) =>
if (dwMap.contains((dw.id))) {
val old = dwMap((dw.id))
val newDW =
Expand All @@ -687,14 +761,6 @@ class Backend @Inject() (implicit
if (entityNames.contains(e.name) && e.searchIndex.isDefined)
} yield e
esRetriever.getTermsResultsMapping(entities, queryTerms)

// withQueryTermsNumberValidation(queryTerms, defaultOTSettings.qValidationLimitNTerms) match {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need this commented code?

// case Success(terms) =>
// esRetriever.getTermsResultsMapping(entities, terms)
// case Failure(error) =>
// Future.failed(error)
// }

}

def search(
Expand Down Expand Up @@ -859,14 +925,14 @@ class Backend @Inject() (implicit
size: Int
): Future[Vector[Similarity]] = {
val table = defaultOTSettings.clickhouse.similarities
logger.info(s"query similarities in table ${table.name}")
logger.debug(s"query similarities in table ${table.name}")

val jointLabels = labels + label
val simQ = QW2V(table.name, categories, jointLabels, threshold, size)
dbRetriever.executeQuery[Long, Query](simQ.existsLabel(label)).flatMap {
case Vector(1) => dbRetriever.executeQuery[Similarity, Query](simQ.query)
case _ =>
logger.info(
logger.debug(
s"This case where the label asked ${label} to the model Word2Vec does not exist" +
s" is ok but nice to capture though"
)
Expand Down Expand Up @@ -911,7 +977,7 @@ class Backend @Inject() (implicit
): Future[Publications] = {
val table = defaultOTSettings.clickhouse.literature
val indexTable = defaultOTSettings.clickhouse.literatureIndex
logger.info(s"query literature ocurrences in table ${table.name}")
logger.debug(s"query literature ocurrences in table ${table.name}")

val pag = Helpers.Cursor.to(cursor).flatMap(_.asOpt[Pagination]).getOrElse(Pagination.mkDefault)

Expand Down Expand Up @@ -948,12 +1014,12 @@ class Backend @Inject() (implicit
case Vector(year) =>
runQuery(year, total)
case _ =>
logger.info(s"Cannot find the earliest year for the publications.")
logger.debug(s"Cannot find the earliest year for the publications.")
runQuery(1900, total)
}

case _ =>
logger.info(s"there is no publications with this set of ids $ids")
logger.debug(s"there is no publications with this set of ids $ids")
Future.successful(Publications.empty())
}
}
Expand Down
Loading