Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add categorical patient level filtering for generic-assay-data-counts #11155

Open
wants to merge 5 commits into
base: demo-rfc80-poc
Choose a base branch
from
+78 −35
Open

Add categorical patient level filtering for generic-assay-data-counts #11155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
16ea817
Add patient level filtering for aggregation
fuzhaoyuan Nov 1, 2024
41b663b
Patient level filtering works for non-NA
fuzhaoyuan Nov 7, 2024
4cbbd30
Categorical patient level filtering & clean up
fuzhaoyuan Nov 8, 2024
d7183e3
Use new generic assay table schema
fuzhaoyuan Nov 14, 2024
e7de357
Fix counting
fuzhaoyuan Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,21 +63,29 @@ private CategorizedGenericAssayDataCountFilter extractGenericAssayDataCountFilte

CategorizedGenericAssayDataCountFilter.Builder builder = CategorizedGenericAssayDataCountFilter.getBuilder();

// TODO: Support patient level profiles and data filtering
List<String> sampleCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
.stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY"))
.map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
.toList();
// TODO: Support data filtering
List<String> sampleNumericalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
.stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE"))
.map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
.toList();
builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
.filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
.toList());
List<String> sampleCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
.stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY"))
.map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
.toList();
builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
.filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
.toList());
// Patient level profiles only have CATEGORICAL datatype for now
List<String> patientCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.PATIENT)
.stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL"))
.map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
.toList();
builder.setPatientCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
.filter(genericAssayDataFilter -> patientCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
.toList());
return builder.build();
}

Expand Down
70 changes: 46 additions & 24 deletions src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@
<include refid="applySampleTreatmentFilter"/>
</if>
<!-- Apply Genomic Data Filter -->
<!-- TODO: add patient level data filtering -->
<if test="studyViewFilterHelper.studyViewFilter.genomicDataFilters != null and !studyViewFilterHelper.studyViewFilter.genomicDataFilters.isEmpty()">
<foreach item="genomicDataFilter" collection="studyViewFilterHelper.studyViewFilter.genomicDataFilters" open="INTERSECT" separator="INTERSECT">
(
Expand All @@ -154,25 +153,9 @@
</foreach>
</if>
<!-- Apply Generic Assay Data Filter -->
<if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters().isEmpty()">
<foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
(
<include refid="numericalGenericAssayDataFilter">
<property name="unique_id" value="sample_unique_id"/>
<property name="table_name" value="generic_assay_data_derived"/>
</include>
)
</foreach>
</if>
<if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters().isEmpty()">
<foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
<include refid="categoricalGenericAssayDataCountFilter">
<property name="unique_id" value="sample_unique_id"/>
<property name="table_name" value="generic_assay_data_derived"/>
</include>
</foreach>
</if>

<if test="studyViewFilterHelper.studyViewFilter.genericAssayDataFilters != null and !studyViewFilterHelper.studyViewFilter.genericAssayDataFilters.isEmpty()">
<include refid="applyGenericAssayDataFilter"/>
</if>
<!-- Apply Clinical Data Filter -->
<if test="studyViewFilterHelper.studyViewFilter.clinicalDataFilters != null and !studyViewFilterHelper.studyViewFilter.clinicalDataFilters.isEmpty()">
<include refid="applyClinicalDataCountFilter"/>
Expand Down Expand Up @@ -549,8 +532,39 @@
</foreach>
</sql>

<sql id="applyGenericAssayDataFilter">
<if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters().isEmpty()">
<foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
(
<include refid="numericalGenericAssayDataFilter">
<property name="type" value="sample"/>
</include>
)
</foreach>
</if>
<if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters().isEmpty()">
<foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
(
<include refid="categoricalGenericAssayDataCountFilter">
<property name="type" value="sample"/>
</include>
)
</foreach>
</if>
-- patient level profile only have categorical for now
<if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters().isEmpty()">
<foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
(
<include refid="categoricalGenericAssayDataCountFilter">
<property name="type" value="patient"/>
</include>
)
</foreach>
</if>
</sql>

<sql id="selectAllGenericAssays">
SELECT sample_unique_id, value, datatype
SELECT sample_unique_id, patient_unique_id, value, datatype
FROM generic_assay_data_derived
WHERE profile_type = #{genericAssayDataFilter.profileType}
AND entity_stable_id = #{genericAssayDataFilter.stableId}
Expand Down Expand Up @@ -647,10 +661,18 @@
</sql>

<sql id="categoricalGenericAssayDataCountFilter">
SELECT ${unique_id}
SELECT sample_unique_id
FROM sample_derived sd
LEFT JOIN (<include refid="selectAllGenericAssays"/>) AS generic_assay_query
ON sd.sample_unique_id = generic_assay_query.sample_unique_id
LEFT JOIN (<include refid="selectAllGenericAssays"/>) AS generic_assay_query
ON
<choose>
<when test="'${type}' == 'sample'">
sd.sample_unique_id = generic_assay_query.sample_unique_id
</when>
<otherwise>
sd.patient_unique_id = generic_assay_query.patient_unique_id
</otherwise>
</choose>
<where>
datatype != 'LIMIT-VALUE'
<foreach item="dataFilterValue" collection="genericAssayDataFilter.values" open=" AND ((" separator=") OR (" close="))">
Expand Down
25 changes: 19 additions & 6 deletions src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,12 +304,17 @@
</select>

<select id="getGenericAssayDataCounts" resultMap="GenericAssayDataCountItemResultMap">
<bind name="profileType" value="genericAssayDataFilters[0].profileType"/>
WITH generic_assay_query AS (
<bind name="profileType" value="genericAssayDataFilters[0].profileType"/>
WITH profile_info AS (
SELECT patient_level
FROM genetic_profile
WHERE stable_id = concat(#{studyViewFilterHelper.studyViewFilter.studyIds[0]}, '_', #{profileType})
),
generic_assay_query AS (
SELECT
entity_stable_id AS stableId,
value,
cast(count(*) AS INTEGER) AS count
cast(count(distinct patient_unique_id) AS INTEGER) AS count
FROM generic_assay_data_derived
<where>
<!-- Table creation in clickhouse.sql has ensured no NA values but extra caution is always appreciated -->
Expand Down Expand Up @@ -339,9 +344,17 @@
<!-- The NA count is specially caculated using total sample count minus non-NA count, therefore
these 2 coalesces are here in case the non-NA subquery returned empty results and we need to provide properties needed to construct the target object -->
SELECT
coalesce((SELECT stableId FROM generic_assay_data_sum LIMIT 1), #{genericAssayDataFilters[0].stableId}) as stableId,
'NA' as value,
cast(((SELECT * FROM (<include refid="getTotalSampleCount"/>)) - coalesce((SELECT gad_count FROM generic_assay_data_sum LIMIT 1), 0)) as INTEGER) as count
coalesce((SELECT stableId FROM generic_assay_data_sum LIMIT 1), #{genericAssayDataFilters[0].stableId}) as stableId,
'NA' as value,
cast((
CASE
WHEN (SELECT patient_level FROM profile_info) = 1 THEN
(SELECT * FROM (<include refid="getTotalPatientCount"/>))
ELSE
(SELECT * FROM (<include refid="getTotalSampleCount"/>))
END
- coalesce((SELECT gad_count FROM generic_assay_data_sum LIMIT 1), 0))
as INTEGER) as count
</select>

<!-- for /mutation-data-counts/fetch (returns GenomicDataCountItem objects) mutation counts pie chart part -->
Expand Down
Loading