Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[risk=no] Sv vcf fb 2 #1470

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open

[risk=no] Sv vcf fb 2 #1470

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
72d7084
addressing sv vcf feedback 1
Oct 9, 2024
eed14ba
generating rolled up location counts
Oct 11, 2024
973e8fb
[DB-1438][risk=no]State/territory heatmap prototype (#1467)
mhigbyflowers Oct 11, 2024
caa45b3
generating rolled up location counts
Oct 11, 2024
718f1d2
generating rolled up location counts
Oct 11, 2024
72c58b0
generating rolled up location counts
Oct 11, 2024
a5dffc1
generating rolled up location counts
Oct 11, 2024
b60d833
generating rolled up location counts
Oct 11, 2024
39753dc
modified query
Oct 11, 2024
80d8f80
modified query
Oct 11, 2024
7d624d5
modified query
Oct 12, 2024
c12ef45
modified query
Oct 12, 2024
a01c419
modified query
Oct 12, 2024
8e77cfc
modified query
Oct 12, 2024
9bee790
modified query
Oct 12, 2024
b9658d2
modified query
Oct 12, 2024
830621b
modified query
Oct 12, 2024
ee8c2a6
modified query
Oct 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion public-api/config/cdr_config_local.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"bigqueryProject": "all-of-us-ehr-dev",
"bigqueryDataset": "synthetic_cdr20180606",
"genomicsProject": "aou-db-prod",
"genomicsDataset": "2022q4r6_genomics",
"genomicsDataset": "2024q3r2_genomics",
"creationTime": "2017-12-26 00:00:00Z",
"releaseNumber": 1,
"numParticipants": 946237,
Expand Down
145 changes: 145 additions & 0 deletions public-api/db-cdr/generate-cdr/generate_criteria_stratum.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,3 +314,148 @@ with y as
group by concept_id,age
order by concept_id asc
"

echo "location counts"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
select distinct c.concept_id, ar.stratum_2 as stratum_1,'location' as stratum_2, 'Procedure', ar.count_value, 3108 from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.achilles_results\` ar join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.concept\` c
on cast(c.concept_id as string)=ar.stratum_1 and analysis_id=3108 join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\` cr on c.concept_id = cr.concept_id
and cr.is_group=0 and cr.is_selectable=1 and cr.type in ('SNOMED', 'ICD9Proc', 'ICD10PCS', 'CPT4', 'ICD9CM') and cr.full_text like '%rank1%' and ar.stratum_3='Procedure' and cr.domain_id='PROCEDURE'
group by c.concept_id,ar.stratum_2,ar.count_value order by concept_id asc"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id,stratum_1,stratum_2, domain, count_value, analysis_id)
WITH state_information AS (
SELECT person_id, c.*
FROM \`${BQ_PROJECT}.${BQ_DATASET}.observation\` ob
JOIN \`${BQ_PROJECT}.${BQ_DATASET}.concept\` c
ON ob.value_source_concept_id = c.concept_id
WHERE observation_source_concept_id = 1585249
)
select concept_id, location, 'location', 'Procedure', cnt, 3108
from
(select ancestor_concept_id as concept_id, si.concept_name as location, count(distinct b.person_id) cnt
from
(select *
from \`${BQ_PROJECT}.${BQ_DATASET}.concept_ancestor\`
where ancestor_concept_id in
(select distinct concept_id
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\`
where type in ('SNOMED', 'ICD9Proc', 'ICD10PCS', 'CPT4', 'ICD9CM')
and domain_id = 'PROCEDURE'
and is_group = 1 and full_text like '%rank1%')) a
join \`${BQ_PROJECT}.${BQ_DATASET}.procedure_occurrence\` b on a.descendant_concept_id = b.procedure_concept_id
join \`${BQ_PROJECT}.${BQ_DATASET}.person\` p on b.person_id=p.person_id
join state_information si on si.person_id = p.person_id
group by 1,2) y
group by concept_id,location,cnt
order by concept_id asc;"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
select c.concept_id, ar.stratum_2 as stratum_1,'location' as stratum_2, 'Condition', ar.count_value, 3108
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.achilles_results\` ar join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.concept\` c
on cast(c.concept_id as string)=ar.stratum_1 and analysis_id=3108 join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\` cr on c.concept_id = cr.concept_id
and cr.is_group=0 and cr.is_selectable=1 and cr.type='SNOMED' and cr.domain_id='CONDITION' and cr.full_text like '%rank1%' and ar.stratum_3='Condition'
group by c.concept_id,ar.stratum_2,ar.count_value order by concept_id asc;"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
WITH state_information AS (
SELECT person_id, c.*
FROM \`${BQ_PROJECT}.${BQ_DATASET}.observation\` ob
JOIN \`${BQ_PROJECT}.${BQ_DATASET}.concept\` c
ON ob.value_source_concept_id = c.concept_id
WHERE observation_source_concept_id = 1585249
)
select concept_id, location, 'biological_sex', 'Condition', cnt, 3101
from
(select ancestor_concept_id as concept_id, si.concept_name as location, count(distinct b.person_id) cnt
from
(select *
from \`${BQ_PROJECT}.${BQ_DATASET}.concept_ancestor\`
where ancestor_concept_id in
(select distinct concept_id
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\`
where type = 'SNOMED'
and domain_id = 'CONDITION'
and is_group = 1 and full_text like '%rank1%')) a
join \`${BQ_PROJECT}.${BQ_DATASET}.condition_occurrence\` b on a.descendant_concept_id = b.condition_concept_id
join \`${BQ_PROJECT}.${BQ_DATASET}.person\` p on p.person_id = b.person_id
join state_information si on p.person_id = si.person_id
group by 1,2) y
group by concept_id,location,cnt
order by concept_id asc"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
select c.concept_id, ar.stratum_2 as stratum_1,'location' as stratum_2, 'Condition', ar.count_value, 3108
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.achilles_results\` ar join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.concept\` c
on cast(c.concept_id as string)=ar.stratum_1 and analysis_id=3108 join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\` cr on c.concept_id = cr.concept_id
and cr.is_group=0 and cr.is_selectable=1 and cr.type='ICD9CM' and cr.domain_id='CONDITION' and cr.full_text like '%rank1%' and ar.stratum_3='Condition'
group by c.concept_id,ar.stratum_2,ar.count_value order by concept_id asc"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
WITH state_information AS (
SELECT person_id, c.*
FROM \`${BQ_PROJECT}.${BQ_DATASET}.observation\` ob
JOIN \`${BQ_PROJECT}.${BQ_DATASET}.concept\` c
ON ob.value_source_concept_id = c.concept_id
WHERE observation_source_concept_id = 1585249
)
select concept_id, location, 'location', 'Condition', cnt, 3101
from
(select ancestor_concept_id as concept_id, si.concept_name as location, count(distinct b.person_id) cnt
from
(select *
from \`${BQ_PROJECT}.${BQ_DATASET}.concept_ancestor\`
where ancestor_concept_id in
(select distinct concept_id
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\`
where type = 'ICD9CM'
and domain_id = 'CONDITION'
and is_group = 1 and full_text like '%rank1%')) a
join \`${BQ_PROJECT}.${BQ_DATASET}.condition_occurrence\` b on a.descendant_concept_id = b.condition_concept_id
join \`${BQ_PROJECT}.${BQ_DATASET}.person\` p on p.person_id = b.person_id
join state_information si on si.person_id = p.person_id
group by 1,2) y
group by concept_id,location,cnt
order by concept_id asc;"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
select c.concept_id, ar.stratum_2 as stratum_1,'location' as stratum_2, 'Condition', ar.count_value, 3108
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.achilles_results\` ar join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.concept\` c
on cast(c.concept_id as string)=ar.stratum_1 and analysis_id=3108 join \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\` cr on c.concept_id = cr.concept_id
and cr.is_group=0 and cr.is_selectable=1 and cr.type='ICD10CM' and cr.domain_id='CONDITION' and cr.full_text like '%rank1%' and ar.stratum_3='Condition'
group by c.concept_id,ar.stratum_2,ar.count_value order by concept_id asc;"

bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"insert into \`$OUTPUT_PROJECT.$OUTPUT_DATASET.criteria_stratum\` (concept_id, stratum_1, stratum_2, domain, count_value, analysis_id)
WITH state_information AS (
SELECT person_id, c.*
FROM \`${BQ_PROJECT}.${BQ_DATASET}.observation\` ob
JOIN \`${BQ_PROJECT}.${BQ_DATASET}.concept\` c
ON ob.value_source_concept_id = c.concept_id
WHERE observation_source_concept_id = 1585249
)
select concept_id, location, 'location', 'Condition', cnt, 3101
from
(select ancestor_concept_id as concept_id, si.concept_name as location, count(distinct b.person_id) cnt
from
(select *
from \`${BQ_PROJECT}.${BQ_DATASET}.concept_ancestor\`
where ancestor_concept_id in
(select distinct concept_id
from \`$OUTPUT_PROJECT.$OUTPUT_DATASET.cb_criteria\`
where type = 'ICD10CM'
and domain_id = 'CONDITION'
and is_group = 1 and full_text like '%rank1%')) a
join \`${BQ_PROJECT}.${BQ_DATASET}.condition_occurrence\` b on a.descendant_concept_id = b.condition_concept_id
join \`${BQ_PROJECT}.${BQ_DATASET}.person\` p on p.person_id = b.person_id
join state_information si on p.person_id = si.person_id
group by 1,2) y
group by concept_id, location,cnt
order by concept_id asc"
46 changes: 17 additions & 29 deletions public-api/src/main/java/org/pmiops/workbench/publicapi/GenomicsController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ public class GenomicsController implements GenomicsApiDelegate {

private static final String genomicRegionRegex = "(?i)([\"]*)(chr([0-9]{1,})*[XYxy]*:{0,}).*";
private static final String variantIdRegex = "(?i)([\"]*)((\\d{1,}|X|Y)-\\d{5,}-[A,C,T,G]{1,}-[A,C,T,G]{1,}).*";
private static final String svVariantIdRegex = "(?i)([1-9]|1[0-9]|2[0-2]|X|Y)-\\d{1,}-[0-9a-fA-F]{4}";
private static final String svVariantIdRegexV7 = "(?i)AoUSVPhase[a-zA-Z0-9]{1,2}\\.chr[1-9XY][0-9]?(?:\\.final_cleanup_)?(BND|DUP|DEL)_chr[1-9XY][0-9]?_\\d+";
private static final String svVariantIdRegexV8 = "(?i)AoUSVPhase[a-zA-Z0-9]{1,2}\\.(BND|DUP|DEL)_chr[1-9XY][0-9]?_shard[0-9][0-9]?_\\d+";

private static final String rsNumberRegex = "(?i)(rs)(\\d{1,})";
private static final String COUNT_SQL_TEMPLATE = "SELECT count(*) as count FROM ${projectId}.${dataSetId}.wgs_variant";

Expand Down Expand Up @@ -609,11 +611,6 @@ public ResponseEntity<Long> getSVVariantSearchResultSize(SVVariantResultSizeRequ
finalSql += HOMOZYGOTE_COUNT_FILTER;
}

System.out.println("######################################################");
System.out.println(finalSql);
System.out.println("######################################################");


QueryJobConfiguration qjc = QueryJobConfiguration.newBuilder(finalSql)
.addNamedParameter("contig", QueryParameterValue.string(contig))
.addNamedParameter("high", QueryParameterValue.int64(high))
Expand All @@ -628,10 +625,6 @@ public ResponseEntity<Long> getSVVariantSearchResultSize(SVVariantResultSizeRequ
Map<String, Integer> rm = bigQueryService.getResultMapper(result);
List<FieldValue> row = result.iterateAll().iterator().next();

System.out.println("######################################################");
System.out.println(rm.get("count"));
System.out.println("######################################################");

return ResponseEntity.ok(bigQueryService.getLong(row, rm.get("count")));
}

Expand Down Expand Up @@ -911,9 +904,9 @@ public ResponseEntity<SVVariantListResponse> searchSVVariants(SearchSVVariantsRe
finalSql += ORDER_BY_CLAUSE;
finalSql += " LIMIT " + rowCount + " OFFSET " + ((Optional.ofNullable(page).orElse(1)-1)*rowCount);

System.out.println("**************************");
System.out.println("*************************************************************");
System.out.println(finalSql);
System.out.println("**************************");
System.out.println("*************************************************************");

QueryJobConfiguration qjc = QueryJobConfiguration.newBuilder(finalSql)
.addNamedParameter("variant_id", QueryParameterValue.string(variant_id))
Expand Down Expand Up @@ -1249,13 +1242,6 @@ public ResponseEntity<VariantListResponse> searchVariants(SearchVariantsRequest

}

System.out.println(contig);
System.out.println("-------------------------------------------------");
System.out.println("-------------------------------------------------");
System.out.println(finalSql);
System.out.println("-------------------------------------------------");
System.out.println("-------------------------------------------------");

VariantListResponse variantListResponse = new VariantListResponse();
variantListResponse.setItems(variantList);
return ResponseEntity.ok(variantListResponse);
Expand Down Expand Up @@ -1483,10 +1469,6 @@ public ResponseEntity<SVGenomicFilters> getSVGenomicFilterOptions(String variant
+ SV_FILTER_OPTION_SQL_TEMPLATE_HOMOZYGOTE_COUNT + searchSqlQuery
+ SV_FILTER_OPTION_SQL_TEMPLATE_UNION;

System.out.println("Test");
System.out.println(finalSql);
System.out.println("Test");

QueryJobConfiguration qjc = QueryJobConfiguration.newBuilder(finalSql)
.addNamedParameter("contig", QueryParameterValue.string(contig))
.addNamedParameter("high", QueryParameterValue.int64(high))
Expand Down Expand Up @@ -1818,7 +1800,18 @@ public static SVGenomicSearchTermType getSVSearchType(String variantSearchTerm,
Long high = 0L;
boolean whereContigFlag = false;

if (searchTerm.matches(genomicRegionRegex)) {

if (searchTerm.matches(svVariantIdRegexV7)) {
// Check if the search term matches variant id v7 pattern
variant_id = searchTerm;
whereVariantIdFlag = true;
searchSql += WHERE_VARIANT_ID;
} else if (searchTerm.matches(svVariantIdRegexV8)) {
// Check if the search term matches variant id v8 pattern
variant_id = searchTerm;
whereVariantIdFlag = true;
searchSql += WHERE_VARIANT_ID;
} else if (searchTerm.matches(genomicRegionRegex)) {
String[] regionTermSplit = new String[0];
if (searchTerm.contains(":")) {
regionTermSplit = searchTerm.split(":");
Expand All @@ -1839,11 +1832,6 @@ public static SVGenomicSearchTermType getSVSearchType(String variantSearchTerm,
System.out.println("Trying to convert bad number.");
}
}
} else if (searchTerm.matches(svVariantIdRegex)) {
// Check if the search term matches variant id pattern
variant_id = searchTerm;
whereVariantIdFlag = true;
searchSql += WHERE_VARIANT_ID;
} else {// Check if the search term matches gene coding pattern
whereGeneFlag = true;
if (variantSearchTerm.startsWith("~")) {
Expand Down
8 changes: 8 additions & 0 deletions public-api/src/main/java/org/pmiops/workbench/service/AchillesAnalysisService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,12 @@ public List<ConceptAnalysis> getConceptAnalyses(List<String> conceptIds, String
addAgeStratum(aa, conceptId, null, 2);
conceptAnalysis.setAgeAnalysis(aa);
} else if (analysisId.equals(CommonStorageEnums.analysisIdFromName(AnalysisIdConstant.LOCATION_ANALYSIS_ID))) {

System.out.println("****************************************************************");
System.out.println("LOCATION ANALYSIS");
System.out.println("****************************************************************");


addLocationStratum(aa, 2, conceptId, null);
conceptAnalysis.setLocationAnalysis(aa);
} else if (analysisId.equals(CommonStorageEnums.analysisIdFromName(AnalysisIdConstant.MEASUREMENT_GENDER_ANALYSIS_ID))) {
Expand Down Expand Up @@ -507,6 +513,7 @@ public void addLocationStratum(Analysis aa, int stratum, String conceptId, List<
uniqueLocationStratums.add(ar.getStratum1());
ar.setAnalysisStratumName(locationStratumNameMap.get(ar.getStratum1()));
} else if (stratum == 2) {
System.out.println(ar.getStratum2());
uniqueLocationStratums.add(ar.getStratum2());
ar.setAnalysisStratumName(locationStratumNameMap.get(ar.getStratum2()));
} else if (stratum == 3) {
Expand Down Expand Up @@ -536,6 +543,7 @@ public void addLocationStratum(Analysis aa, int stratum, String conceptId, List<

// Add missing location strata
for (String missingLocation : completeLocationStratumList) {
System.out.println("Missing");
AchillesResult missingResult = null;
if (aa.getAnalysisId() == CommonStorageEnums.analysisIdFromName(AnalysisIdConstant.LOCATION_ANALYSIS_ID)) {
missingResult = achillesMapper.makeCopyAchillesResult(aa.getAnalysisId(), domainConceptId, null, conceptId, missingLocation, null, null, null, 20L, 20L);
Expand Down
24 changes: 12 additions & 12 deletions public-ui/src/app/app-routing.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,10 @@ export const AppRoutingComponent: React.FunctionComponent = () => {
path="/variants"
component={() =>
GenomicViewComponent({
selectionId: 2,
selectionId: 1,
routeData: {
title: "Variants",
breadcrumb: { value: "Variants" },
title: "Genomic Variants",
breadcrumb: { value: "Genomic Variants" },
},
})
}
Expand All @@ -119,10 +119,10 @@ export const AppRoutingComponent: React.FunctionComponent = () => {
path="/variants/:search"
component={() =>
GenomicViewComponent({
selectionId: 2,
selectionId: 1,
routeData: {
title: "Variants",
breadcrumb: { value: "Variants" },
title: "Genomic Variants",
breadcrumb: { value: "Genomic Variants" },
},
})
}
Expand All @@ -131,10 +131,10 @@ export const AppRoutingComponent: React.FunctionComponent = () => {
path="/structural-variants"
component={() =>
GenomicViewComponent({
selectionId: 4,
selectionId: 2,
routeData: {
title: "Variants",
breadcrumb: { value: "Variants" },
title: "Genomic Variants",
breadcrumb: { value: "Genomic Variants" },
},
})
}
Expand All @@ -143,10 +143,10 @@ export const AppRoutingComponent: React.FunctionComponent = () => {
path="/structural-variants/:search"
component={() =>
GenomicViewComponent({
selectionId: 4,
selectionId: 2,
routeData: {
title: "Variants",
breadcrumb: { value: "Variants" },
title: "Genomic Variants",
breadcrumb: { value: "Genomic Variants" },
},
})
}
Expand Down
16 changes: 8 additions & 8 deletions public-ui/src/app/data-browser/databrowser-routing.module.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@ const routes: Routes = [
component: AppRouting,
canActivate: [IsSafeGuard],
data: {
title: "Variants",
title: "Genomic Variants",
breadcrumb: {
value: "Variants",
value: "Genomic Variants",
},
},
},
Expand All @@ -93,9 +93,9 @@ const routes: Routes = [
component: AppRouting,
canActivate: [IsSafeGuard],
data: {
title: "Variants",
title: "Genomic Variants",
breadcrumb: {
value: "Variants",
value: "Genomic Variants",
},
},
},
Expand All @@ -104,9 +104,9 @@ const routes: Routes = [
component: AppRouting,
canActivate: [IsSafeGuard],
data: {
title: "Variants",
title: "Genomic Variants",
breadcrumb: {
value: "Variants",
value: "Genomic Variants",
},
},
},
Expand All @@ -115,9 +115,9 @@ const routes: Routes = [
component: AppRouting,
canActivate: [IsSafeGuard],
data: {
title: "Variants",
title: "Genomic Variants",
breadcrumb: {
value: "Variants",
value: "Genomic Variants",
},
},
},
Expand Down
Loading