From 0901d4c242da622944bd8ee2e459cd7d1307c0ba Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Thu, 20 Jan 2022 14:21:25 +0100 Subject: [PATCH] feat: Ability to get unique dimension values filtered by above filters --- .../components/chart-configurator.tsx | 6 +- app/graphql/queries/data-cubes.graphql | 12 +- app/graphql/query-hooks.ts | 40 +++++- app/graphql/resolver-types.ts | 36 ++++- app/graphql/resolvers.ts | 48 +++++-- app/graphql/schema.graphql | 17 ++- app/pages/api/graphql.ts | 30 +++-- app/rdf/queries.ts | 41 +++--- app/rdf/query-dimension-values.ts | 126 ++++++++++++++++-- 9 files changed, 288 insertions(+), 68 deletions(-) diff --git a/app/configurator/components/chart-configurator.tsx b/app/configurator/components/chart-configurator.tsx index 77ae975db4..f435a3193f 100644 --- a/app/configurator/components/chart-configurator.tsx +++ b/app/configurator/components/chart-configurator.tsx @@ -62,7 +62,11 @@ export const ChartConfigurator = ({ }) => { const locale = useLocale(); const [{ data }] = useDataCubeMetadataWithComponentValuesQuery({ - variables: { iri: state.dataSet, locale }, + variables: { + iri: state.dataSet, + locale, + filters: state.chartConfig.filters, + }, }); if (data?.dataCubeByIri) { diff --git a/app/graphql/queries/data-cubes.graphql b/app/graphql/queries/data-cubes.graphql index 620b2fe7fb..497e40c656 100644 --- a/app/graphql/queries/data-cubes.graphql +++ b/app/graphql/queries/data-cubes.graphql @@ -36,7 +36,7 @@ fragment dimensionMetaData on Dimension { iri label isKeyDimension - values + values(filters: $filters) unit ... on TemporalDimension { timeUnit @@ -44,7 +44,12 @@ fragment dimensionMetaData on Dimension { } } -query DataCubePreview($iri: String!, $locale: String!, $latest: Boolean) { +query DataCubePreview( + $iri: String! + $locale: String! + $latest: Boolean + $filters: Filters +) { dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) { iri title @@ -101,6 +106,7 @@ query DataCubeMetadataWithComponentValues( $iri: String! $locale: String! $latest: Boolean + $filters: Filters ) { dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) { iri @@ -120,6 +126,7 @@ query DimensionValues( $dimensionIri: String! $locale: String! $latest: Boolean + $filters: Filters ) { dataCubeByIri(iri: $dataCubeIri, locale: $locale, latest: $latest) { dimensionByIri(iri: $dimensionIri) { @@ -133,6 +140,7 @@ query TemporalDimensionValues( $dimensionIri: String! $locale: String! $latest: Boolean + $filters: Filters ) { dataCubeByIri(iri: $dataCubeIri, locale: $locale, latest: $latest) { dimensionByIri(iri: $dimensionIri) { diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index cab9eca10a..dbfddba383 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -103,6 +103,11 @@ export type Dimension = { }; +export type DimensionValuesArgs = { + filters?: Maybe; +}; + + export type Measure = Dimension & { __typename: 'Measure'; @@ -114,6 +119,11 @@ export type Measure = Dimension & { values: Array; }; + +export type MeasureValuesArgs = { + filters?: Maybe; +}; + export type NominalDimension = Dimension & { __typename: 'NominalDimension'; iri: Scalars['String']; @@ -125,6 +135,11 @@ export type NominalDimension = Dimension & { }; +export type NominalDimensionValuesArgs = { + filters?: Maybe; +}; + + export type ObservationsQuery = { __typename: 'ObservationsQuery'; /** Observations with their values parsed to native JS types */ @@ -147,6 +162,11 @@ export type OrdinalDimension = Dimension & { values: Array; }; + +export type OrdinalDimensionValuesArgs = { + filters?: Maybe; +}; + export type Query = { __typename: 'Query'; dataCubeByIri?: Maybe; @@ -162,6 +182,7 @@ export type QueryDataCubeByIriArgs = { locale?: Maybe; iri: Scalars['String']; latest?: Maybe; + filters?: Maybe; }; @@ -209,6 +230,11 @@ export type TemporalDimension = Dimension & { values: Array; }; + +export type TemporalDimensionValuesArgs = { + filters?: Maybe; +}; + export enum TimeUnit { Year = 'Year', Month = 'Month', @@ -244,6 +270,7 @@ export type DataCubePreviewQueryVariables = Exact<{ iri: Scalars['String']; locale: Scalars['String']; latest?: Maybe; + filters?: Maybe; }>; @@ -287,6 +314,7 @@ export type DataCubeMetadataWithComponentValuesQueryVariables = Exact<{ iri: Scalars['String']; locale: Scalars['String']; latest?: Maybe; + filters?: Maybe; }>; @@ -312,6 +340,7 @@ export type DimensionValuesQueryVariables = Exact<{ dimensionIri: Scalars['String']; locale: Scalars['String']; latest?: Maybe; + filters?: Maybe; }>; @@ -334,6 +363,7 @@ export type TemporalDimensionValuesQueryVariables = Exact<{ dimensionIri: Scalars['String']; locale: Scalars['String']; latest?: Maybe; + filters?: Maybe; }>; @@ -404,7 +434,7 @@ export const DimensionMetaDataFragmentDoc = gql` iri label isKeyDimension - values + values(filters: $filters) unit ... on TemporalDimension { timeUnit @@ -446,7 +476,7 @@ export function useDataCubesQuery(options: Omit({ query: DataCubesDocument, ...options }); }; export const DataCubePreviewDocument = gql` - query DataCubePreview($iri: String!, $locale: String!, $latest: Boolean) { + query DataCubePreview($iri: String!, $locale: String!, $latest: Boolean, $filters: Filters) { dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) { iri title @@ -509,7 +539,7 @@ export function useDataCubeMetadataQuery(options: Omit({ query: DataCubeMetadataDocument, ...options }); }; export const DataCubeMetadataWithComponentValuesDocument = gql` - query DataCubeMetadataWithComponentValues($iri: String!, $locale: String!, $latest: Boolean) { + query DataCubeMetadataWithComponentValues($iri: String!, $locale: String!, $latest: Boolean, $filters: Filters) { dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) { iri title @@ -528,7 +558,7 @@ export function useDataCubeMetadataWithComponentValuesQuery(options: Omit({ query: DataCubeMetadataWithComponentValuesDocument, ...options }); }; export const DimensionValuesDocument = gql` - query DimensionValues($dataCubeIri: String!, $dimensionIri: String!, $locale: String!, $latest: Boolean) { + query DimensionValues($dataCubeIri: String!, $dimensionIri: String!, $locale: String!, $latest: Boolean, $filters: Filters) { dataCubeByIri(iri: $dataCubeIri, locale: $locale, latest: $latest) { dimensionByIri(iri: $dimensionIri) { ...dimensionMetaData @@ -541,7 +571,7 @@ export function useDimensionValuesQuery(options: Omit({ query: DimensionValuesDocument, ...options }); }; export const TemporalDimensionValuesDocument = gql` - query TemporalDimensionValues($dataCubeIri: String!, $dimensionIri: String!, $locale: String!, $latest: Boolean) { + query TemporalDimensionValues($dataCubeIri: String!, $dimensionIri: String!, $locale: String!, $latest: Boolean, $filters: Filters) { dataCubeByIri(iri: $dataCubeIri, locale: $locale, latest: $latest) { dimensionByIri(iri: $dimensionIri) { ... on TemporalDimension { diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 14e0b9c8f5..856f19b697 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -108,6 +108,11 @@ export type Dimension = { }; +export type DimensionValuesArgs = { + filters?: Maybe; +}; + + export type Measure = Dimension & { __typename?: 'Measure'; @@ -119,6 +124,11 @@ export type Measure = Dimension & { values: Array; }; + +export type MeasureValuesArgs = { + filters?: Maybe; +}; + export type NominalDimension = Dimension & { __typename?: 'NominalDimension'; iri: Scalars['String']; @@ -130,6 +140,11 @@ export type NominalDimension = Dimension & { }; +export type NominalDimensionValuesArgs = { + filters?: Maybe; +}; + + export type ObservationsQuery = { __typename?: 'ObservationsQuery'; /** Observations with their values parsed to native JS types */ @@ -152,6 +167,11 @@ export type OrdinalDimension = Dimension & { values: Array; }; + +export type OrdinalDimensionValuesArgs = { + filters?: Maybe; +}; + export type Query = { __typename?: 'Query'; dataCubeByIri?: Maybe; @@ -167,6 +187,7 @@ export type QueryDataCubeByIriArgs = { locale?: Maybe; iri: Scalars['String']; latest?: Maybe; + filters?: Maybe; }; @@ -214,6 +235,11 @@ export type TemporalDimension = Dimension & { values: Array; }; + +export type TemporalDimensionValuesArgs = { + filters?: Maybe; +}; + export enum TimeUnit { Year = 'Year', Month = 'Month', @@ -395,7 +421,7 @@ export type DimensionResolvers, ParentType, ContextType>; scaleType?: Resolver, ParentType, ContextType>; isKeyDimension?: Resolver; - values?: Resolver, ParentType, ContextType>; + values?: Resolver, ParentType, ContextType, RequireFields>; }>; export interface DimensionValueScalarConfig extends GraphQLScalarTypeConfig { @@ -412,7 +438,7 @@ export type MeasureResolvers, ParentType, ContextType>; scaleType?: Resolver, ParentType, ContextType>; isKeyDimension?: Resolver; - values?: Resolver, ParentType, ContextType>; + values?: Resolver, ParentType, ContextType, RequireFields>; __isTypeOf?: IsTypeOfResolverFn; }>; @@ -422,7 +448,7 @@ export type NominalDimensionResolvers, ParentType, ContextType>; scaleType?: Resolver, ParentType, ContextType>; isKeyDimension?: Resolver; - values?: Resolver, ParentType, ContextType>; + values?: Resolver, ParentType, ContextType, RequireFields>; __isTypeOf?: IsTypeOfResolverFn; }>; @@ -444,7 +470,7 @@ export type OrdinalDimensionResolvers, ParentType, ContextType>; scaleType?: Resolver, ParentType, ContextType>; isKeyDimension?: Resolver; - values?: Resolver, ParentType, ContextType>; + values?: Resolver, ParentType, ContextType, RequireFields>; __isTypeOf?: IsTypeOfResolverFn; }>; @@ -469,7 +495,7 @@ export type TemporalDimensionResolvers, ParentType, ContextType>; scaleType?: Resolver, ParentType, ContextType>; isKeyDimension?: Resolver; - values?: Resolver, ParentType, ContextType>; + values?: Resolver, ParentType, ContextType, RequireFields>; __isTypeOf?: IsTypeOfResolverFn; }>; diff --git a/app/graphql/resolvers.ts b/app/graphql/resolvers.ts index fcfe7c13b9..40ac7a2dc5 100644 --- a/app/graphql/resolvers.ts +++ b/app/graphql/resolvers.ts @@ -1,9 +1,13 @@ import { ascending, descending } from "d3"; +import DataLoader from "dataloader"; import fuzzaldrin from "fuzzaldrin-plus"; import { GraphQLJSONObject } from "graphql-type-json"; +import { Filters } from "../configurator"; import { DimensionValue } from "../domain/data"; import { parseLocaleString } from "../locales/locales"; +import { Loaders } from "../pages/api/graphql"; import { + createCubeDimensionValuesLoader, getCube, getCubeDimensions, getCubeObservations, @@ -235,23 +239,47 @@ const DataCube: DataCubeResolvers = { }, }; -const dimensionResolvers = { +const getDimensionValuesLoader = ( + loaders: Loaders, + filters?: Filters | null +): DataLoader => { + let loader: typeof loaders.dimensionValues | undefined; + const filterKey = filters ? JSON.stringify(filters) : undefined; + if (filterKey && filters) { + let existingLoader = loaders.filteredDimensionValues.get(filterKey); + if (!existingLoader) { + loader = new DataLoader(createCubeDimensionValuesLoader(filters)); + loaders.filteredDimensionValues.set(filterKey, loader); + return loader; + } else { + return existingLoader; + } + } else { + return loaders.dimensionValues; + } +}; + +const mkDimensionResolvers = (debugName: string) => ({ iri: ({ data: { iri } }: ResolvedDimension) => iri, label: ({ data: { name } }: ResolvedDimension) => name, isKeyDimension: ({ data: { isKeyDimension } }: ResolvedDimension) => isKeyDimension, unit: ({ data: { unit } }: ResolvedDimension) => unit ?? null, scaleType: ({ data: { scaleType } }: ResolvedDimension) => scaleType ?? null, - values: async (parent: ResolvedDimension, _: {}, { loaders }: any) => { - const values: Array = await loaders.dimensionValues.load( - parent - ); + values: async ( + parent: ResolvedDimension, + { filters }: { filters?: Filters | null }, + { loaders }: { loaders: Loaders } + ) => { + // Different loader if we have filters or not + const loader = getDimensionValuesLoader(loaders, filters); + const values: Array = await loader.load(parent); // TODO min max are now just `values` with 2 elements. Handle properly! return values.sort((a, b) => ascending(a.value ?? undefined, b.value ?? undefined) ); }, -}; +}); export const resolvers: Resolvers = { Filters: GraphQLJSONObject, @@ -302,17 +330,17 @@ export const resolvers: Resolvers = { }, }, NominalDimension: { - ...dimensionResolvers, + ...mkDimensionResolvers("nominal"), }, OrdinalDimension: { - ...dimensionResolvers, + ...mkDimensionResolvers("ordinal"), }, TemporalDimension: { - ...dimensionResolvers, + ...mkDimensionResolvers("temporal"), timeUnit: ({ data: { timeUnit } }: ResolvedDimension) => timeUnit!, timeFormat: ({ data: { timeFormat } }: ResolvedDimension) => timeFormat!, }, Measure: { - ...dimensionResolvers, + ...mkDimensionResolvers("measure"), }, }; diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index e5957cc504..3b34e6b9e2 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -49,7 +49,7 @@ interface Dimension { unit: String scaleType: String isKeyDimension: Boolean! - values: [DimensionValue!]! + values(filters: Filters): [DimensionValue!]! } type NominalDimension implements Dimension { @@ -58,7 +58,7 @@ type NominalDimension implements Dimension { unit: String scaleType: String isKeyDimension: Boolean! - values: [DimensionValue!]! + values(filters: Filters): [DimensionValue!]! } type OrdinalDimension implements Dimension { @@ -67,7 +67,7 @@ type OrdinalDimension implements Dimension { unit: String scaleType: String isKeyDimension: Boolean! - values: [DimensionValue!]! + values(filters: Filters): [DimensionValue!]! } enum TimeUnit { @@ -88,7 +88,7 @@ type TemporalDimension implements Dimension { unit: String scaleType: String isKeyDimension: Boolean! - values: [DimensionValue!]! + values(filters: Filters): [DimensionValue!]! } type Measure implements Dimension { @@ -97,7 +97,7 @@ type Measure implements Dimension { unit: String scaleType: String isKeyDimension: Boolean! - values: [DimensionValue!]! + values(filters: Filters): [DimensionValue!]! } type DataCubeResult { @@ -136,7 +136,12 @@ type DatasetCount { # The "Query" type is special: it lists all of the available queries that # clients can execute, along with the return type for each. type Query { - dataCubeByIri(locale: String, iri: String!, latest: Boolean = true): DataCube + dataCubeByIri( + locale: String + iri: String! + latest: Boolean = true + filters: Filters + ): DataCube dataCubes( locale: String query: String diff --git a/app/pages/api/graphql.ts b/app/pages/api/graphql.ts index 96ab756cd8..d08630b9e0 100644 --- a/app/pages/api/graphql.ts +++ b/app/pages/api/graphql.ts @@ -3,6 +3,7 @@ import configureCors from "cors"; import DataLoader from "dataloader"; import "global-agent/bootstrap"; import { NextApiRequest, NextApiResponse } from "next"; +import { Filters } from "../../configurator"; import { resolvers } from "../../graphql/resolvers"; import typeDefs from "../../graphql/schema.graphql"; import { runMiddleware } from "../../lib/run-middleware"; @@ -14,6 +15,23 @@ import { const cors = configureCors(); +const makeLoaders = (req: any) => { + return { + dimensionValues: new DataLoader(createCubeDimensionValuesLoader(), { + cacheKeyFn: (dim) => dim.dimension.path?.value, + }), + filteredDimensionValues: new Map>(), + themes: new DataLoader( + createThemeLoader({ locale: req.headers["accept-language"] }) + ), + organizations: new DataLoader( + createOrganizationLoader({ locale: req.headers["accept-language"] }) + ), + } as const; +}; + +export type Loaders = ReturnType; + const server = new ApolloServer({ typeDefs, resolvers, @@ -22,17 +40,7 @@ const server = new ApolloServer({ return err; }, context: ({ req }) => ({ - loaders: { - dimensionValues: new DataLoader(createCubeDimensionValuesLoader(), { - cacheKeyFn: (dim) => dim.dimension.path?.value, - }), - themes: new DataLoader( - createThemeLoader({ locale: req.headers["accept-language"] }) - ), - organizations: new DataLoader( - createOrganizationLoader({ locale: req.headers["accept-language"] }) - ), - }, + loaders: makeLoaders(req), }), // Enable playground in production introspection: true, diff --git a/app/rdf/queries.ts b/app/rdf/queries.ts index a7223e620c..6f4da5fa8f 100644 --- a/app/rdf/queries.ts +++ b/app/rdf/queries.ts @@ -197,9 +197,11 @@ export const getCube = async ({ export const getCubeDimensions = async ({ cube, locale, + filters, }: { cube: Cube; locale: string; + filters?: Filters | null; }): Promise => { try { const dimensions = cube.dimensions.filter( @@ -238,23 +240,21 @@ export const getCubeDimensions = async ({ }; export const createCubeDimensionValuesLoader = - () => async (dimensions: readonly ResolvedDimension[]) => { + (filters?: Filters) => async (dimensions: readonly ResolvedDimension[]) => { const result: DimensionValue[][] = []; for (const dimension of dimensions) { - const dimensionValues = await getCubeDimensionValues(dimension); + const dimensionValues = await getCubeDimensionValues(dimension, filters); result.push(dimensionValues); } return result; }; -export const getCubeDimensionValues = async ({ - dimension, - cube, - locale, - data, -}: ResolvedDimension): Promise => { +export const getCubeDimensionValues = async ( + { dimension, cube, locale, data }: ResolvedDimension, + filters?: Filters +): Promise => { if (data.dataKind === "Time") { // return interpolateTimeValues({ // dataType: data.dataType, @@ -267,8 +267,8 @@ export const getCubeDimensionValues = async ({ } if ( - dimension.minInclusive !== undefined && - dimension.maxInclusive !== undefined + typeof dimension.minInclusive !== "undefined" && + typeof dimension.maxInclusive !== "undefined" ) { const min = parseObservationValue({ value: dimension.minInclusive }) ?? 0; const max = parseObservationValue({ value: dimension.maxInclusive }) ?? 0; @@ -283,6 +283,7 @@ export const getCubeDimensionValues = async ({ dimension, cube, locale, + filters, }); }; @@ -321,27 +322,33 @@ const groupLabelsPerValue = ({ return [...grouped.values()]; }; -const dimensionIsVersioned = (dimension: CubeDimension) => +export const dimensionIsVersioned = (dimension: CubeDimension) => dimension.out(ns.schema.version)?.value ? true : false; const getCubeDimensionValuesWithLabels = async ({ dimension, cube, locale, + filters, }: { dimension: CubeDimension; cube: Cube; locale: string; + filters?: Filters; }): Promise => { const load = async () => { const loaders = [ - () => dimension.in || [], + !filters ? () => dimension.in || [] : undefined, () => - loadDimensionValues({ - datasetIri: cube.term, - dimensionIri: dimension.path, - }), - ]; + loadDimensionValues( + { + datasetIri: cube.term, + dimension, + cube, + }, + filters + ), + ].filter(truthy); for (const loader of loaders) { const dimensionValues = await loader(); diff --git a/app/rdf/query-dimension-values.ts b/app/rdf/query-dimension-values.ts index 002ea3601e..aa0a80fbe9 100644 --- a/app/rdf/query-dimension-values.ts +++ b/app/rdf/query-dimension-values.ts @@ -1,29 +1,133 @@ import { SELECT } from "@tpluscode/sparql-builder"; import { Literal, NamedNode, Term } from "rdf-js"; -import { cube } from "./namespace"; +import { Filters } from "../configurator"; +import { cube as cubeNs } from "./namespace"; import { sparqlClient } from "./sparql-client"; +import { Cube, CubeDimension } from "rdf-cube-view-query"; +import { dimensionIsVersioned } from "./queries"; +import * as ns from "./namespace"; interface DimensionValue { value: Literal | NamedNode; } +/** + * Formats a filter value into the right format given + * the datatype of the dimension + * + * Seems a bit fragile, we should find a way to directly add the ^^xsd + * given the datatype instead of handling everycase + */ +const formatFilterValue = ( + value: string | number, + dimension: CubeDimension +) => { + if (!dimension.datatype) { + return `<${value}>`; + } else { + // Seems fragile + if (dimension.datatype.value === ns.xsd.gYear.value) { + return `"${value}"^^xsd:gYear`; + } else if (dimension.datatype.value === ns.xsd.date.value) { + return `"${value}"^^xsd:date`; + } else if (dimension.datatype.value === ns.xsd.dateTime.value) { + return `"${value}"^^xsd:dateTime`; + } else { + return `"${value}"`; + } + } +}; + +const formatFilterIntoSparqlFilter = ( + filter: Filters[string], + dimension: CubeDimension, + versioned: boolean, + index: number +) => { + const suffix = versioned ? "_unversioned" : ""; + if (filter.type === "single") { + return `FILTER ( (?dimension${suffix}${index} = ${formatFilterValue( + filter.value, + dimension + )}) )`; + } else if (filter.type === "multi") { + return `FILTER ( (?dimension${suffix}${index} in (${Object.keys( + filter.values + ) + .map((x) => formatFilterValue(x, dimension)) + .join(",")}) ) )`; + } else { + return ""; + } +}; + /** * Load dimension values. + * + * Filters on other dimensions can be passed. + * */ -export async function loadDimensionValues({ - datasetIri, - dimensionIri, -}: { - datasetIri: Term | undefined; - dimensionIri: Term | undefined; -}): Promise> { - const query = SELECT.DISTINCT`?value`.WHERE` - ${datasetIri} ${cube.observationSet} ?observationSet . - ?observationSet ${cube.observation} ?observation . +export async function loadDimensionValues( + { + datasetIri, + dimension, + cube, + }: { + datasetIri: Term | undefined; + dimension: CubeDimension; + cube: Cube; + }, + filters?: Filters +): Promise> { + const dimensionIri = dimension.path; + + let filterList = filters ? Object.entries(filters) : []; + filterList = filterList.slice( + 0, + filterList.findIndex(([iri]) => iri == dimensionIri?.value) + 1 + ); + + let query = SELECT.DISTINCT`?value`.WHERE` + ${datasetIri} ${cubeNs.observationSet} ?observationSet . + ?observationSet ${cubeNs.observation} ?observation . ?observation ${dimensionIri} ?value . + ${ + filters + ? filterList + .map(([iri, value], idx) => { + const filterDimension = cube.dimensions.find( + (d) => d.path?.value === iri + ); + if ( + !filterDimension || + value.type === "range" || + dimensionIri?.value === iri + ) { + return ""; + } + const versioned = filterDimension + ? dimensionIsVersioned(filterDimension) + : false; + return `${ + versioned + ? `?dimension${idx} ?dimension_unversioned${idx}.` + : "" + } + ?observation <${iri}> ?dimension${idx}. + ${formatFilterIntoSparqlFilter( + value, + filterDimension, + versioned, + idx + )}`; + }) + .join("\n") + : "" + } `; let result: Array = []; + console.log(query.build()); try { result = (await query.execute(sparqlClient.query, {