diff --git a/app/.env.development b/app/.env.development index a29f1f3be..7c72a3786 100644 --- a/app/.env.development +++ b/app/.env.development @@ -1,5 +1,5 @@ DATABASE_URL=postgres://postgres:password@localhost:5432/visualization_tool -SPARQL_ENDPOINT=https://lindas.admin.ch/query SPARQL_GEO_ENDPOINT=https://geo.ld.admin.ch/query -SPARQL_EDITOR=https://lindas.admin.ch/sparql/ +SPARQL_ENDPOINT=https://int.lindas.admin.ch/query +SPARQL_EDITOR=https://int.lindas.admin.ch/sparql/ GRAPHQL_ENDPOINT=/api/graphql diff --git a/app/charts/area/areas-state.tsx b/app/charts/area/areas-state.tsx index 13a24c81a..0b074819e 100644 --- a/app/charts/area/areas-state.tsx +++ b/app/charts/area/areas-state.tsx @@ -18,6 +18,7 @@ import { stackOrderReverse, sum, } from "d3"; +import { sortBy } from "lodash"; import { ReactNode, useMemo } from "react"; import { AreaFields } from "../../configurator"; import { @@ -29,6 +30,7 @@ import { Observation } from "../../domain/data"; import { sortByIndex } from "../../lib/array"; import { estimateTextWidth } from "../../lib/estimate-text-width"; import { useLocale } from "../../locales/use-locale"; +import { makeOrdinalDimensionSorter } from "../../utils/sorting-values"; import { BRUSH_BOTTOM_SPACE } from "../shared/brush"; import { getLabelWithUnit, @@ -184,32 +186,57 @@ const useAreasState = ({ const segmentSortingType = fields.segment?.sorting?.sortingType; const segmentSortingOrder = fields.segment?.sorting?.sortingOrder; - const segmentsOrderedByName = Array.from( - new Set(sortedData.map((d) => getSegment(d))) - ).sort((a, b) => - segmentSortingOrder === "asc" - ? a.localeCompare(b, locale) - : b.localeCompare(a, locale) - ); + const segments = useMemo(() => { + const getSegmentsOrderedByName = () => + Array.from(new Set(sortedData.map((d) => getSegment(d)))).sort((a, b) => + segmentSortingOrder === "asc" + ? a.localeCompare(b, locale) + : b.localeCompare(a, locale) + ); + + const getSegmentsOrderedByTotalValue = () => + [ + ...rollup( + sortedData, + (v) => sum(v, (x) => getY(x)), + (x) => getSegment(x) + ), + ] + .sort((a, b) => + segmentSortingOrder === "asc" + ? ascending(a[1], b[1]) + : descending(a[1], b[1]) + ) + .map((d) => d[0]); + + const getSegmentsOrderedByPosition = () => { + const segments = Array.from( + new Set(sortedData.map((d) => getSegment(d))) + ); + const sorter = dimension ? makeOrdinalDimensionSorter(dimension) : null; + return sorter ? sortBy(segments, sorter) : segments; + }; - const segmentsOrderedByTotalValue = [ - ...rollup( - sortedData, - (v) => sum(v, (x) => getY(x)), - (x) => getSegment(x) - ), - ] - .sort((a, b) => - segmentSortingOrder === "asc" - ? ascending(a[1], b[1]) - : descending(a[1], b[1]) - ) - .map((d) => d[0]); - - const segments = - segmentSortingType === "byDimensionLabel" - ? segmentsOrderedByName - : segmentsOrderedByTotalValue; + const dimension = dimensions.find( + (dim) => dim.iri === fields.segment?.componentIri + ); + if (dimension?.__typename === "OrdinalDimension") { + return getSegmentsOrderedByPosition(); + } + + return segmentSortingType === "byDimensionLabel" + ? getSegmentsOrderedByName() + : getSegmentsOrderedByTotalValue(); + }, [ + dimensions, + fields.segment?.componentIri, + getSegment, + getY, + locale, + segmentSortingOrder, + segmentSortingType, + sortedData, + ]); // Stack order const stackOrder = diff --git a/app/charts/bar/bars-state.tsx b/app/charts/bar/bars-state.tsx index 6d1d731e5..1d5680264 100644 --- a/app/charts/bar/bars-state.tsx +++ b/app/charts/bar/bars-state.tsx @@ -76,7 +76,10 @@ const useBarsState = ({ }, [data, getX, getY, sortingType, sortingOrder]); // segments - const segments = [...new Set(sortedData.map((d) => getSegment(d)))]; + const segments = useMemo( + () => [...new Set(sortedData.map((d) => getSegment(d)))], + [getSegment, sortedData] + ); const colors = scaleOrdinal(getPalette(fields.segment?.palette)).domain( segments ); diff --git a/app/charts/column/columns-grouped-state.tsx b/app/charts/column/columns-grouped-state.tsx index 8f5b56e3e..d5d56a0f1 100644 --- a/app/charts/column/columns-grouped-state.tsx +++ b/app/charts/column/columns-grouped-state.tsx @@ -16,6 +16,7 @@ import { ScaleTime, sum, } from "d3"; +import { sortBy } from "lodash"; import React, { ReactNode, useMemo } from "react"; import { ColumnFields, SortingOrder, SortingType } from "../../configurator"; import { @@ -26,6 +27,7 @@ import { import { Observation } from "../../domain/data"; import { sortByIndex } from "../../lib/array"; import { useLocale } from "../../locales/use-locale"; +import { makeOrdinalDimensionSorter } from "../../utils/sorting-values"; import { getLabelWithUnit, useOptionalNumericVariable, @@ -143,32 +145,58 @@ const useGroupedColumnsState = ({ // segments const segmentSortingType = fields.segment?.sorting?.sortingType; const segmentSortingOrder = fields.segment?.sorting?.sortingOrder; - const segmentsOrderedByName = Array.from( - new Set(sortedData.map((d) => getSegment(d))) - ).sort((a, b) => - segmentSortingOrder === "asc" - ? a.localeCompare(b, locale) - : b.localeCompare(a, locale) - ); - const segmentsOrderedByTotalValue = [ - ...rollup( - sortedData, - (v) => sum(v, (x) => getY(x)), - (x) => getSegment(x) - ), - ] - .sort((a, b) => - segmentSortingOrder === "asc" - ? ascending(a[1], b[1]) - : descending(a[1], b[1]) - ) - .map((d) => d[0]); + const segments = useMemo(() => { + const getSegmentsOrderedByName = () => + Array.from(new Set(sortedData.map((d) => getSegment(d)))).sort((a, b) => + segmentSortingOrder === "asc" + ? a.localeCompare(b, locale) + : b.localeCompare(a, locale) + ); + + const dimension = dimensions.find( + (d) => d.iri === fields.segment?.componentIri + ); + + const getSegmentsOrderedByPosition = () => { + const segments = Array.from( + new Set(sortedData.map((d) => getSegment(d))) + ); + if (!dimension) { + return segments; + } + const sorter = makeOrdinalDimensionSorter(dimension); + return sortBy(segments, sorter); + }; - const segments = - segmentSortingType === "byDimensionLabel" - ? segmentsOrderedByName - : segmentsOrderedByTotalValue; + const getSegmentsOrderedByTotalValue = () => + [ + ...rollup( + sortedData, + (v) => sum(v, (x) => getY(x)), + (x) => getSegment(x) + ), + ] + .sort((a, b) => + segmentSortingOrder === "asc" + ? ascending(a[1], b[1]) + : descending(a[1], b[1]) + ) + .map((d) => d[0]); + if (dimension?.__typename === "OrdinalDimension") { + return getSegmentsOrderedByPosition(); + } + return segmentSortingType === "byDimensionLabel" + ? getSegmentsOrderedByName() + : getSegmentsOrderedByTotalValue(); + }, [ + getSegment, + getY, + locale, + segmentSortingOrder, + segmentSortingType, + sortedData, + ]); // Map ordered segments to colors const colors = scaleOrdinal(); diff --git a/app/charts/column/columns-stacked-state.tsx b/app/charts/column/columns-stacked-state.tsx index 61e857d65..042185e73 100644 --- a/app/charts/column/columns-stacked-state.tsx +++ b/app/charts/column/columns-stacked-state.tsx @@ -21,7 +21,7 @@ import { stackOrderReverse, sum, } from "d3"; -import { keyBy } from "lodash"; +import { keyBy, sortBy } from "lodash"; import React, { ReactNode, useCallback, useMemo } from "react"; import { ColumnFields, SortingOrder, SortingType } from "../../configurator"; import { @@ -32,6 +32,7 @@ import { Observation } from "../../domain/data"; import { DimensionMetaDataFragment } from "../../graphql/query-hooks"; import { sortByIndex } from "../../lib/array"; import { useLocale } from "../../locales/use-locale"; +import { makeOrdinalDimensionSorter } from "../../utils/sorting-values"; import { getLabelWithUnit, getWideData, @@ -176,36 +177,57 @@ const useColumnsStackedState = ({ const segmentSortingOrder = fields.segment?.sorting?.sortingOrder; const segments = useMemo(() => { - const segmentsOrderedByName = Array.from( - new Set(sortedData.map((d) => getSegment(d))) - ).sort((a, b) => - segmentSortingOrder === "asc" - ? a.localeCompare(b, locale) - : b.localeCompare(a, locale) + const getSegmentsOrderedByName = () => + Array.from(new Set(sortedData.map((d) => getSegment(d)))).sort((a, b) => + segmentSortingOrder === "asc" + ? a.localeCompare(b, locale) + : b.localeCompare(a, locale) + ); + + const dimension = dimensions.find( + (d) => d.iri === fields.segment?.componentIri ); + const getSegmentsOrderedByPosition = () => { + const segments = Array.from( + new Set(sortedData.map((d) => getSegment(d))) + ); + if (!dimension) { + return segments; + } + const sorter = makeOrdinalDimensionSorter(dimension); + return sortBy(segments, sorter); + }; + + const getSegmentsOrderedByTotalValue = () => + [ + ...rollup( + sortedData, + (v) => sum(v, (x) => getY(x)), + (x) => getSegment(x) + ), + ] + .sort((a, b) => + segmentSortingOrder === "asc" + ? ascending(a[1], b[1]) + : descending(a[1], b[1]) + ) + .map((d) => d[0]); + + if (dimension?.__typename === "OrdinalDimension") { + return getSegmentsOrderedByPosition(); + } - const segmentsOrderedByTotalValue = [ - ...rollup( - sortedData, - (v) => sum(v, (x) => getY(x)), - (x) => getSegment(x) - ), - ] - .sort((a, b) => - segmentSortingOrder === "asc" - ? ascending(a[1], b[1]) - : descending(a[1], b[1]) - ) - .map((d) => d[0]); return segmentSortingType === "byDimensionLabel" - ? segmentsOrderedByName - : segmentsOrderedByTotalValue; + ? getSegmentsOrderedByName() + : getSegmentsOrderedByTotalValue(); }, [ - sortedData, + dimensions, segmentSortingType, + sortedData, getSegment, segmentSortingOrder, locale, + fields.segment?.componentIri, getY, ]); diff --git a/app/charts/column/columns-state.tsx b/app/charts/column/columns-state.tsx index 033b7a4ba..d0a5609c4 100644 --- a/app/charts/column/columns-state.tsx +++ b/app/charts/column/columns-state.tsx @@ -13,6 +13,7 @@ import { scaleTime, ScaleTime, } from "d3"; +import { sortBy } from "lodash"; import { ReactNode, useMemo } from "react"; import { ColumnFields, SortingOrder, SortingType } from "../../configurator"; import { @@ -23,6 +24,7 @@ import { } from "../../configurator/components/ui-helpers"; import { Observation } from "../../domain/data"; import { TimeUnit } from "../../graphql/query-hooks"; +import { makeOrdinalDimensionSorter } from "../../utils/sorting-values"; import { getLabelWithUnit, useOptionalNumericVariable, @@ -184,10 +186,21 @@ const useColumnsState = ({ yScale.range([chartHeight, 0]); // segments - const segments = Array.from(new Set(sortedData.map((d) => getSegment(d)))); - const colors = scaleOrdinal(getPalette(fields.segment?.palette)).domain( - segments - ); + const segments = useMemo(() => { + return Array.from(new Set(sortedData.map(getSegment))); + }, [getSegment, sortedData]); + const sortedSegments = useMemo(() => { + const rawSegments = getPalette(fields.segment?.palette); + const segmentDimension = dimensions.find( + (d) => d.iri === fields.segment?.componentIri + ); + const sorter = + segmentDimension?.__typename === "OrdinalDimension" + ? makeOrdinalDimensionSorter(segmentDimension) + : null; + return sorter ? sortBy(rawSegments, sorter) : rawSegments; + }, [fields.segment?.palette, fields.segment?.componentIri, dimensions]); + const colors = scaleOrdinal(sortedSegments).domain(segments); // Tooltip const getAnnotationInfo = (datum: Observation): TooltipInfo => { diff --git a/app/charts/line/lines-state.tsx b/app/charts/line/lines-state.tsx index aba8d19f2..97f7aa3c8 100644 --- a/app/charts/line/lines-state.tsx +++ b/app/charts/line/lines-state.tsx @@ -11,6 +11,7 @@ import { ScaleTime, scaleTime, } from "d3"; +import { sortBy } from "lodash"; import { ReactNode, useMemo } from "react"; import { LineFields } from "../../configurator"; import { @@ -22,6 +23,7 @@ import { Observation } from "../../domain/data"; import { sortByIndex } from "../../lib/array"; import { estimateTextWidth } from "../../lib/estimate-text-width"; import { useTheme } from "../../themes"; +import { makeOrdinalDimensionSorter } from "../../utils/sorting-values"; import { BRUSH_BOTTOM_SPACE } from "../shared/brush"; import { getLabelWithUnit, @@ -173,9 +175,20 @@ const useLinesState = ({ const yAxisLabel = getLabelWithUnit(yMeasure); // segments - const segments = [...new Set(sortedData.map(getSegment))].sort((a, b) => - ascending(a, b) - ); + const segments = useMemo(() => { + const segments = [...new Set(sortedData.map(getSegment))].sort((a, b) => + ascending(a, b) + ); + const dimension = dimensions.find( + (d) => d.iri === fields?.segment?.componentIri + ); + if (dimension?.__typename === "OrdinalDimension") { + const sorter = makeOrdinalDimensionSorter(dimension); + return sortBy(segments, sorter); + } + return segments; + }, [dimensions, fields?.segment?.componentIri, getSegment, sortedData]); + // Map ordered segments to colors const colors = scaleOrdinal(); const segmentDimension = dimensions.find( diff --git a/app/charts/shared/legend-color.tsx b/app/charts/shared/legend-color.tsx index d8ea9391b..50b122764 100644 --- a/app/charts/shared/legend-color.tsx +++ b/app/charts/shared/legend-color.tsx @@ -10,7 +10,7 @@ import { ColumnsState } from "../column/columns-state"; import { LinesState } from "../line/lines-state"; import { PieState } from "../pie/pie-state"; import { ScatterplotState } from "../scatterplot/scatterplot-state"; -import { useChartState } from "./use-chart-state"; +import { ColorsChartState, useChartState } from "./use-chart-state"; import { useInteractiveFilters } from "./use-interactive-filters"; type LegendSymbol = "square" | "line" | "circle"; @@ -73,16 +73,7 @@ export const LegendColor = memo(function LegendColor({ }: { symbol: LegendSymbol; }) { - const { colors } = useChartState() as - | BarsState - | GroupedBarsState - | ColumnsState - | StackedColumnsState - | GroupedColumnsState - | LinesState - | AreasState - | ScatterplotState - | PieState; + const { colors } = useChartState() as ColorsChartState; return ( = T extends { [k in R]: any } + ? T + : never; + +export type ColorsChartState = Has; export const ChartContext = createContext(undefined); export const useChartState = () => { diff --git a/app/components/chart-preview.tsx b/app/components/chart-preview.tsx index 9dfaaf786..01d31e6fe 100644 --- a/app/components/chart-preview.tsx +++ b/app/components/chart-preview.tsx @@ -1,36 +1,18 @@ import { Trans } from "@lingui/macro"; import * as React from "react"; import { Box, Flex, Text } from "theme-ui"; -import { ChartAreasVisualization } from "../charts/area/chart-area"; -import { ChartBarsVisualization } from "../charts/bar/chart-bar"; -import { ChartColumnsVisualization } from "../charts/column/chart-column"; -import { ChartLinesVisualization } from "../charts/line/chart-lines"; -import { ChartMapVisualization } from "../charts/map/chart-map"; -import { ChartPieVisualization } from "../charts/pie/chart-pie"; -import { ChartScatterplotVisualization } from "../charts/scatterplot/chart-scatterplot"; import { ChartDataFilters } from "../charts/shared/chart-data-filters"; import { useQueryFilters } from "../charts/shared/chart-helpers"; import { InteractiveFiltersProvider } from "../charts/shared/use-interactive-filters"; import useSyncInteractiveFilters from "../charts/shared/use-sync-interactive-filters"; -import { ChartTableVisualization } from "../charts/table/chart-table"; -import { - ChartConfig, - isAreaConfig, - isBarConfig, - isColumnConfig, - isLineConfig, - isMapConfig, - isPieConfig, - isScatterPlotConfig, - isTableConfig, - useConfiguratorState, -} from "../configurator"; +import { ChartConfig, useConfiguratorState } from "../configurator"; import { useDataCubeMetadataQuery } from "../graphql/query-hooks"; import { DataCubePublicationStatus } from "../graphql/resolver-types"; import { useLocale } from "../locales/use-locale"; import { ChartErrorBoundary } from "./chart-error-boundary"; import { ChartFiltersList } from "./chart-filters-list"; import { ChartFootnotes } from "./chart-footnotes"; +import GenericChart from "./common-chart"; import DebugPanel from "./debug-panel"; import { HintRed } from "./hint"; @@ -166,64 +148,11 @@ const Chart = ({ chartConfig, }); - return ( - <> - {/* CHARTS */} - {isColumnConfig(chartConfig) && ( - - )} - {isBarConfig(chartConfig) && ( - - )} - {isLineConfig(chartConfig) && ( - - )} - {isAreaConfig(chartConfig) && ( - - )} - {isScatterPlotConfig(chartConfig) && ( - - )} - {isPieConfig(chartConfig) && ( - - )} - {isTableConfig(chartConfig) && ( - - )} - {isMapConfig(chartConfig) && ( - - )} - - ); + const props = { + dataSet, + chartConfig: chartConfig, + queryFilters: queryFilters, + }; + + return ; }; diff --git a/app/components/chart-published.tsx b/app/components/chart-published.tsx index 6a4e7693d..29fe6db24 100644 --- a/app/components/chart-published.tsx +++ b/app/components/chart-published.tsx @@ -2,21 +2,12 @@ import { Trans } from "@lingui/macro"; import * as React from "react"; import { useEffect } from "react"; import { Box, Flex, Text } from "theme-ui"; -import { ChartAreasVisualization } from "../charts/area/chart-area"; -import { ChartBarsVisualization } from "../charts/bar/chart-bar"; -import { ChartColumnsVisualization } from "../charts/column/chart-column"; -import { ChartLinesVisualization } from "../charts/line/chart-lines"; -import { ChartMapVisualization } from "../charts/map/chart-map"; -import { ChartPieVisualization } from "../charts/pie/chart-pie"; -import { ChartScatterplotVisualization } from "../charts/scatterplot/chart-scatterplot"; import { ChartDataFilters } from "../charts/shared/chart-data-filters"; -import { QueryFilters, useQueryFilters } from "../charts/shared/chart-helpers"; import { isUsingImputation } from "../charts/shared/imputation"; import { InteractiveFiltersProvider, useInteractiveFilters, } from "../charts/shared/use-interactive-filters"; -import { ChartTableVisualization } from "../charts/table/chart-table"; import { ChartConfig, Meta } from "../configurator"; import { parseDate } from "../configurator/components/ui-helpers"; import { useDataCubeMetadataQuery } from "../graphql/query-hooks"; @@ -24,6 +15,7 @@ import { DataCubePublicationStatus } from "../graphql/resolver-types"; import { useLocale } from "../locales/use-locale"; import { ChartErrorBoundary } from "./chart-error-boundary"; import { ChartFootnotes } from "./chart-footnotes"; +import GenericChart from "./common-chart"; import { HintBlue, HintRed } from "./hint"; export const ChartPublished = ({ @@ -164,59 +156,7 @@ const ChartWithInteractiveFilters = ({ chartConfig={chartConfig} /> )} - + ); }; - -const getChart = ({ - chartConfig, - ...props -}: { - dataSetIri: string; - chartConfig: ChartConfig; - queryFilters: QueryFilters; -}) => { - switch (chartConfig.chartType) { - case "column": - return ; - case "bar": - return ; - case "line": - return ; - case "area": - return ; - case "scatterplot": - return ( - - ); - case "pie": - return ; - case "table": - return ; - case "map": - return ; - default: - const _exhaustiveCheck: never = chartConfig; - return _exhaustiveCheck; - } -}; - -const Chart = ({ - dataSet, - chartConfig, -}: { - dataSet: string; - chartConfig: ChartConfig; -}) => { - // Combine filters from config + interactive filters - const queryFilters = useQueryFilters({ - chartConfig, - }); - - return getChart({ - dataSetIri: dataSet, - chartConfig, - queryFilters, - }); -}; diff --git a/app/components/common-chart.tsx b/app/components/common-chart.tsx new file mode 100644 index 000000000..1d20799c4 --- /dev/null +++ b/app/components/common-chart.tsx @@ -0,0 +1,55 @@ +import { ChartAreasVisualization } from "../charts/area/chart-area"; +import { ChartBarsVisualization } from "../charts/bar/chart-bar"; +import { ChartColumnsVisualization } from "../charts/column/chart-column"; +import { ChartLinesVisualization } from "../charts/line/chart-lines"; +import { ChartMapVisualization } from "../charts/map/chart-map"; +import { ChartPieVisualization } from "../charts/pie/chart-pie"; +import { ChartScatterplotVisualization } from "../charts/scatterplot/chart-scatterplot"; +import { ChartTableVisualization } from "../charts/table/chart-table"; +import { ChartConfig } from "../configurator"; +import { useQueryFilters } from "../charts/shared/chart-helpers"; + +const GenericChart = ({ + dataSet, + chartConfig, +}: { + dataSet: string; + chartConfig: ChartConfig; +}) => { + // Combine filters from config + interactive filters + const queryFilters = useQueryFilters({ + chartConfig, + }); + + const props = { + dataSetIri: dataSet, + chartConfig, + queryFilters, + }; + + switch (chartConfig.chartType) { + case "column": + return ; + case "bar": + return ; + case "line": + return ; + case "area": + return ; + case "scatterplot": + return ( + + ); + case "pie": + return ; + case "table": + return ; + case "map": + return ; + default: + const _exhaustiveCheck: never = chartConfig; + return _exhaustiveCheck; + } +}; + +export default GenericChart; diff --git a/app/components/form.tsx b/app/components/form.tsx index 2531c6f31..0a5bc0456 100644 --- a/app/components/form.tsx +++ b/app/components/form.tsx @@ -176,7 +176,13 @@ export const Select = ({ const restOptions = options.filter((o) => !o.isNoneValue); if (sortOptions) { - restOptions.sort((a, b) => a.label.localeCompare(b.label, locale)); + restOptions.sort((a, b) => { + if (a.position !== undefined && b.position !== undefined) { + return a.position < b.position; + } else { + return a.label.localeCompare(b.label, locale); + } + }); } return [...noneOptions, ...restOptions]; diff --git a/app/configurator/components/filters.tsx b/app/configurator/components/filters.tsx index 111584fd3..bf4cec0e8 100644 --- a/app/configurator/components/filters.tsx +++ b/app/configurator/components/filters.tsx @@ -19,6 +19,7 @@ import { HierarchyValue, useHierarchicalDimensionValuesQuery, } from "../../utils/dimension-hierarchy"; +import { valueComparator } from "../../utils/sorting-values"; import { EditorIntervalBrush } from "../interactive-filters/editor-time-interval-brush"; import { Accordion, AccordionContent, AccordionSummary } from "./Accordion"; import { @@ -274,9 +275,7 @@ export const DimensionValuesSingleFilter = ({ const sortedDimensionValues = useMemo(() => { return dimension?.values - ? [...dimension.values].sort((a, b) => - a.label.localeCompare(b.label, locale) - ) + ? [...dimension.values].sort(valueComparator(locale)) : []; }, [dimension?.values, locale]); diff --git a/app/configurator/config-form.tsx b/app/configurator/config-form.tsx index b4ed87da8..db098f08d 100644 --- a/app/configurator/config-form.tsx +++ b/app/configurator/config-form.tsx @@ -23,6 +23,7 @@ import { FIELD_VALUE_NONE } from "./constants"; export type Option = { value: string | $FixMe; label: string | $FixMe; + position?: number; isNoneValue?: boolean; disabled?: boolean; }; diff --git a/app/domain/data.ts b/app/domain/data.ts index bebdaa5bf..b5dd4ffd9 100644 --- a/app/domain/data.ts +++ b/app/domain/data.ts @@ -12,7 +12,11 @@ export type RawObservation = Record; export type ObservationValue = string | number | null; -export type DimensionValue = { value: string | number; label: string }; +export type DimensionValue = { + value: string | number; + label: string; + position?: number; +}; export type Observation = Record; diff --git a/app/graphql/resolvers.ts b/app/graphql/resolvers.ts index f9084c426..3672a9319 100644 --- a/app/graphql/resolvers.ts +++ b/app/graphql/resolvers.ts @@ -284,7 +284,10 @@ const mkDimensionResolvers = (debugName: string) => ({ const values: Array = await loader.load(parent); // TODO min max are now just `values` with 2 elements. Handle properly! return values.sort((a, b) => - ascending(a.value ?? undefined, b.value ?? undefined) + ascending( + a.position ?? a.value ?? undefined, + b.position ?? b.value ?? undefined + ) ); }, }); @@ -333,6 +336,10 @@ export const resolvers: Resolvers = { return "GeoShapesDimension"; } + if (scaleType === "Ordinal") { + return "OrdinalDimension"; + } + return "NominalDimension"; }, }, diff --git a/app/rdf/batch-load.ts b/app/rdf/batch-load.ts new file mode 100644 index 000000000..2f9bee6a5 --- /dev/null +++ b/app/rdf/batch-load.ts @@ -0,0 +1,44 @@ +import { SparqlQueryExecutable } from "@tpluscode/sparql-builder/lib"; +import { groups } from "d3"; +import { NamedNode, Term } from "rdf-js"; +import ParsingClient from "sparql-http-client/ParsingClient"; +import { sparqlClient } from "./sparql-client"; + +const BATCH_SIZE = 500; + +export default async function batchLoad< + TReturn extends unknown, + TId extends Term | NamedNode = Term +>({ + ids, + client = sparqlClient, + buildQuery, + batchSize = BATCH_SIZE, +}: { + ids: TId[]; + client?: ParsingClient; + buildQuery: (values: TId[], key: number) => SparqlQueryExecutable; + batchSize?: number; +}): Promise { + const batched = groups(ids, (_, i) => Math.floor(i / batchSize)); + + const results = await Promise.all( + batched.map(async ([key, values]) => { + const query = buildQuery(values, key); + + try { + return (await query.execute(client.query, { + operation: "postUrlencoded", + })) as unknown as TReturn[]; + } catch (e) { + console.log( + `Error while querying. First ID of ${ids.length}: <${ids[0].value}>` + ); + console.error(e); + return []; + } + }) + ); + + return results.flat(); +} diff --git a/app/rdf/parse.ts b/app/rdf/parse.ts index 5bcd007c8..903e0988d 100644 --- a/app/rdf/parse.ts +++ b/app/rdf/parse.ts @@ -96,6 +96,21 @@ const timeFormats = new Map([ [ns.xsd.dateTime.value, "%Y-%m-%dT%H:%M:%S"], ]); +export const getScaleType = ( + dim: CubeDimension +): ResolvedDimension["data"]["scaleType"] => { + const scaleTypeTerm = dim.out(ns.qudt.scaleType).term; + return scaleTypeTerm?.equals(ns.qudt.NominalScale) + ? "Nominal" + : scaleTypeTerm?.equals(ns.qudt.OrdinalScale) + ? "Ordinal" + : scaleTypeTerm?.equals(ns.qudt.RatioScale) + ? "Ratio" + : scaleTypeTerm?.equals(ns.qudt.IntervalScale) + ? "Interval" + : undefined; +}; + export const parseCubeDimension = ({ dim, cube, @@ -113,7 +128,6 @@ export const parseCubeDimension = ({ const timeUnitTerm = dim .out(ns.cube`meta/dataKind`) .out(ns.time.unitType).term; - const scaleTypeTerm = dim.out(ns.qudt.scaleType).term; let dataType = dim.datatype; let hasUndefinedValues = false; @@ -191,15 +205,7 @@ export const parseCubeDimension = ({ : undefined, timeUnit: timeUnits.get(timeUnitTerm?.value ?? ""), timeFormat: timeFormats.get(dataType?.value ?? ""), - scaleType: scaleTypeTerm?.equals(ns.qudt.NominalScale) - ? "Nominal" - : scaleTypeTerm?.equals(ns.qudt.OrdinalScale) - ? "Ordinal" - : scaleTypeTerm?.equals(ns.qudt.RatioScale) - ? "Ratio" - : scaleTypeTerm?.equals(ns.qudt.IntervalScale) - ? "Interval" - : undefined, + scaleType: getScaleType(dim), }, }; }; diff --git a/app/rdf/queries.ts b/app/rdf/queries.ts index a646d9972..e5dd4d600 100644 --- a/app/rdf/queries.ts +++ b/app/rdf/queries.ts @@ -1,6 +1,4 @@ -// import { sparql } from "@tpluscode/rdf-string"; -// import { descending } from "d3"; -import { descending, group, index, rollup } from "d3"; +import { descending, group, index } from "d3"; import { Cube, CubeDimension, @@ -25,12 +23,14 @@ import truthy from "../utils/truthy"; import * as ns from "./namespace"; import { getQueryLocales, + getScaleType, isCubePublished, parseCube, parseCubeDimension, } from "./parse"; import { loadDimensionValues } from "./query-dimension-values"; import { loadResourceLabels } from "./query-labels"; +import { loadResourcePositions } from "./query-positions"; import { loadUnversionedResources } from "./query-sameas"; import { loadUnitLabels } from "./query-unit-labels"; @@ -277,41 +277,6 @@ export const getCubeDimensionValues = async ( }); }; -// const getTemporalDimensionValues = ({ -// dimension, -// min, -// max, -// }: ResolvedDimension) => { - -// }; - -type ValueWithLabel = { value: string; label: string }; - -const groupLabelsPerValue = ({ - values, - locale, -}: { - values: ValueWithLabel[]; - locale: string; -}): ValueWithLabel[] => { - const grouped = rollup( - values, - (vals) => { - const label = vals - .map((v) => v.label) - .sort((a, b) => a.localeCompare(b, locale)) - .join(" / "); - return { - value: vals[0].value, - label: label, - }; - }, - (d) => d.value - ); - - return [...grouped.values()]; -}; - export const dimensionIsVersioned = (dimension: CubeDimension) => dimension.out(ns.schema.version)?.value ? true : false; @@ -364,11 +329,6 @@ const getCubeDimensionValuesWithLabels = async ({ console.warn( `WARNING: dimension with mixed literals and named nodes <${dimension.path?.value}>` ); - - // console.log(`Named:`); - // console.log(dimensionValueNamedNodes); - // console.log(`Literal:`); - // console.log(dimensionValueLiterals); } if (namedNodes.length === 0 && literals.length === 0) { @@ -383,29 +343,32 @@ const getCubeDimensionValuesWithLabels = async ({ * If the dimension is versioned, we're loading the "unversioned" values to store in the config, * so cubes can be upgraded to newer versions without the filters breaking. */ - if (namedNodes.length > 0) { - const [labels, unversioned] = await Promise.all([ + const scaleType = getScaleType(dimension); + const [labels, positions, unversioned] = await Promise.all([ loadResourceLabels({ ids: namedNodes, locale }), + scaleType === "Ordinal" ? loadResourcePositions({ ids: namedNodes }) : [], dimensionIsVersioned(dimension) ? loadUnversionedResources({ ids: namedNodes }) : [], ]); const labelLookup = new Map( - labels.map(({ iri, label }) => { - return [iri.value, label?.value]; - }) + labels.map(({ iri, label }) => [iri.value, label?.value]) + ); + + const positionsLookup = new Map( + positions.map(({ iri, position }) => [iri.value, position?.value]) ); const unversionedLookup = new Map( - unversioned.map(({ iri, sameAs }) => { - return [iri.value, sameAs?.value]; - }) + unversioned.map(({ iri, sameAs }) => [iri.value, sameAs?.value]) ); return namedNodes.map((iri) => { + const pos = positionsLookup.get(iri.value); return { + position: pos !== undefined ? parseInt(pos, 10) : undefined, value: unversionedLookup.get(iri.value) ?? iri.value, label: labelLookup.get(iri.value) ?? "", }; @@ -459,8 +422,6 @@ export const getCubeObservations = async ({ ? buildFilters({ cube, view: cubeView, filters, locale }) : []; - // let observationFilters = []; - /** * Add labels to named dimensions */ diff --git a/app/rdf/query-labels.ts b/app/rdf/query-labels.ts index 6bb69b854..d9df32c9c 100644 --- a/app/rdf/query-labels.ts +++ b/app/rdf/query-labels.ts @@ -1,13 +1,12 @@ import { schema } from "@tpluscode/rdf-ns-builders"; import { sparql } from "@tpluscode/rdf-string"; +import { TemplateResult } from "@tpluscode/rdf-string/lib/TemplateResult"; import { SELECT } from "@tpluscode/sparql-builder"; -import { groups } from "d3"; import { Literal, NamedNode } from "rdf-js"; import ParsingClient from "sparql-http-client/ParsingClient"; import { getQueryLocales } from "./parse"; import { sparqlClient } from "./sparql-client"; - -const BATCH_SIZE = 500; +import batchLoad from "./batch-load"; interface ResourceLabel { iri: NamedNode; @@ -41,13 +40,20 @@ export const makeLocalesFilter = ( `; }; +const buildResourceLabelsQuery = ( + values: NamedNode[], + localesFilter: TemplateResult<$Unexpressable> +) => { + return SELECT.DISTINCT`?iri ?label`.WHERE` + values ?iri { + ${values} + } + ${localesFilter} + `; +}; + /** * Load labels for a list of IDs (e.g. dimension values) - * - * @param ids IDs as rdf-js Terms - * @param client SparqlClient - * - * @todo Add language filter */ export async function loadResourceLabels({ ids, @@ -60,33 +66,11 @@ export async function loadResourceLabels({ labelTerm?: NamedNode; client?: ParsingClient; }): Promise { - // We query in batches because we might run into "413 – Error: Payload Too Large" - const batched = groups(ids, (_, i) => Math.floor(i / BATCH_SIZE)); - const localesFilter = makeLocalesFilter("?iri", labelTerm, "?label", locale); - const results = await Promise.all( - batched.map(async ([key, values]) => { - const query = SELECT.DISTINCT`?iri ?label`.WHERE` - values ?iri { - ${values} - } - ${localesFilter} - `; - - let result: ResourceLabel[] = []; - try { - result = (await query.execute(client.query, { - operation: "postUrlencoded", - })) as unknown as ResourceLabel[]; - } catch (e) { - console.log( - `Could not query labels. First ID of ${ids.length}: <${ids[0].value}>` - ); - console.error(e); - } - return result; - }) - ); - - return results.flat(); + return batchLoad({ + ids, + client, + buildQuery: (values: NamedNode[]) => + buildResourceLabelsQuery(values, localesFilter), + }); } diff --git a/app/rdf/query-positions.ts b/app/rdf/query-positions.ts new file mode 100644 index 000000000..470191f8b --- /dev/null +++ b/app/rdf/query-positions.ts @@ -0,0 +1,38 @@ +import { schema } from "@tpluscode/rdf-ns-builders"; +import { SELECT } from "@tpluscode/sparql-builder"; +import { NamedNode, Literal } from "rdf-js"; +import ParsingClient from "sparql-http-client/ParsingClient"; +import batchLoad from "./batch-load"; +import { sparqlClient } from "./sparql-client"; + +interface ResourcePosition { + iri: NamedNode; + position?: Literal; +} + +const buildResourcePositionQuery = (values: NamedNode[]) => { + return SELECT.DISTINCT`?iri ?position`.WHERE` + values ?iri { + ${values} + } + ?iri ${schema.position} ?position. + `; +}; + +/** + * Load positions for a list of IDs (e.g. dimension values) + * Dimension must be an ordinal one + */ +export async function loadResourcePositions({ + ids, + client = sparqlClient, +}: { + ids: NamedNode[]; + client?: ParsingClient; +}) { + return batchLoad({ + ids, + client, + buildQuery: (values) => buildResourcePositionQuery(values), + }); +} diff --git a/app/rdf/query-sameas.ts b/app/rdf/query-sameas.ts index fe7fa84fe..933157900 100644 --- a/app/rdf/query-sameas.ts +++ b/app/rdf/query-sameas.ts @@ -1,24 +1,26 @@ import { schema } from "@tpluscode/rdf-ns-builders"; import { SELECT } from "@tpluscode/sparql-builder"; -import { groups } from "d3"; import { NamedNode } from "rdf-js"; import ParsingClient from "sparql-http-client/ParsingClient"; +import batchLoad from "./batch-load"; import { sparqlClient } from "./sparql-client"; -const BATCH_SIZE = 500; - interface UnversionedResource { iri: NamedNode; sameAs?: NamedNode; } +const buildUnversionedResourceQuery = (values: NamedNode[]) => { + return SELECT.DISTINCT`?iri ?sameAs`.WHERE` + values ?iri { + ${values} + } + ?iri ${schema.sameAs} ?sameAs . + `; +}; + /** - * Load labels for a list of IDs (e.g. dimension values) - * - * @param ids IDs as rdf-js Terms - * @param client SparqlClient - * - * @todo Add language filter + * Load unversioned resources for a list of IDs (e.g. dimension values) */ export async function loadUnversionedResources({ ids, @@ -27,33 +29,5 @@ export async function loadUnversionedResources({ ids: NamedNode[]; client?: ParsingClient; }): Promise { - // We query in batches because we might run into "413 – Error: Payload Too Large" - const batched = groups(ids, (_, i) => Math.floor(i / BATCH_SIZE)); - - const results = await Promise.all( - batched.map(async ([key, values]) => { - const query = SELECT.DISTINCT`?iri ?sameAs`.WHERE` - values ?iri { - ${values} - } - ?iri ${schema.sameAs} ?sameAs . - `; - - let result: UnversionedResource[] = []; - try { - // console.log(query.build()); - result = ((await query.execute(client.query, { - operation: "postUrlencoded", - })) as unknown) as UnversionedResource[]; - } catch (e) { - console.log( - `Could not query unversioned IRIs. First ID of ${ids.length}: <${ids[0].value}>` - ); - console.error(e); - } - return result; - }) - ); - - return results.flat(); + return batchLoad({ ids, client, buildQuery: buildUnversionedResourceQuery }); } diff --git a/app/rdf/query-unit-labels.ts b/app/rdf/query-unit-labels.ts index fa70e291f..e8f5bf95f 100644 --- a/app/rdf/query-unit-labels.ts +++ b/app/rdf/query-unit-labels.ts @@ -1,40 +1,17 @@ import { SELECT } from "@tpluscode/sparql-builder"; -import { groups } from "d3"; import { Term } from "rdf-js"; import ParsingClient from "sparql-http-client/ParsingClient"; import { sparqlClient } from "./sparql-client"; import * as ns from "./namespace"; - -const BATCH_SIZE = 500; +import batchLoad from "./batch-load"; interface ResourceLabel { iri: Term; label?: Term; } -/** - * Load labels for a list of unit IDs - * - * @param ids IDs as rdf-js Terms - * @param client SparqlClient - * - * @todo Add language filter - */ -export async function loadUnitLabels({ - ids, - locale = "en", - client = sparqlClient, -}: { - ids: Term[]; - locale?: string; - client?: ParsingClient; -}): Promise { - // We query in batches because we might run into "413 – Error: Payload Too Large" - const batched = groups(ids, (_, i) => Math.floor(i / BATCH_SIZE)); - - const results = await Promise.all( - batched.map(async ([key, values]) => { - const query = SELECT.DISTINCT`?iri ?label`.WHERE` +const buildUnitLabelsQuery = (values: Term[], locale: string) => { + return SELECT.DISTINCT`?iri ?label`.WHERE` values ?iri { ${values} } @@ -48,22 +25,23 @@ export async function loadUnitLabels({ FILTER ( lang(?rdfsLabel) = "${locale}" ) `; +}; - let result: ResourceLabel[] = []; - try { - // console.log(query.build()); - result = (await query.execute(client.query, { - operation: "postUrlencoded", - })) as unknown as ResourceLabel[]; - } catch (e) { - console.log( - `Could not query labels. First ID of ${ids.length}: <${ids[0].value}>` - ); - console.error(e); - } - return result; - }) - ); - - return results.flat(); +/** + * Load labels for a list of unit IDs + */ +export async function loadUnitLabels({ + ids, + locale = "en", + client = sparqlClient, +}: { + ids: Term[]; + locale?: string; + client?: ParsingClient; +}): Promise { + return batchLoad({ + ids, + client, + buildQuery: (values: Term[]) => buildUnitLabelsQuery(values, locale), + }); } diff --git a/app/utils/dimension-hierarchy.ts b/app/utils/dimension-hierarchy.ts index eae19b155..fd7b0fb45 100644 --- a/app/utils/dimension-hierarchy.ts +++ b/app/utils/dimension-hierarchy.ts @@ -1,10 +1,11 @@ import { ascending } from "d3"; +import flowRight from "lodash/flowRight"; import { useMemo } from "react"; -import { useClient } from "urql"; import { DataCubeObservationsQuery, useDataCubeObservationsQuery, } from "../graphql/query-hooks"; +import { defaultSorter, makeOrdinalDimensionSorter } from "./sorting-values"; export type DimensionHierarchy = { dimensionIri: string; @@ -155,12 +156,15 @@ const addValuesToTree = ( type TreeSorters = Record< string, - ({ value, label }: { value?: string; label?: string }) => string | undefined + ({ + value, + label, + }: { + value?: string; + label?: string; + }) => string | number | undefined >; -const defaultSorter = ({ value, label }: { value?: string; label?: string }) => - label || value; - /** Recursively sorts tree children */ export const sortTree = (tree: HierarchyValue[], sorters?: TreeSorters) => { const dimensionIri = tree[0].dimensionIri; @@ -210,7 +214,6 @@ export const makeDimensionValuesTree = ({ ); addValuesToTree(tree, valuesByLabels); - sortTree(tree, sorters); return tree; @@ -262,8 +265,6 @@ export const useHierarchicalDimensionValuesQuery = ({ locale: string; dataSetIri: string; }) => { - const client = useClient(); - const path = useMemo( () => getHierarchyDimensionPath(dimensionIri, hierarchy), [dimensionIri] @@ -281,10 +282,23 @@ export const useHierarchicalDimensionValuesQuery = ({ if (fetching || !datacubeResponse || !datacubeResponse.dataCubeByIri) { return { data: undefined, fetching }; } else { + const dataCubeData = datacubeResponse.dataCubeByIri; + const sorters = Object.fromEntries( + dataCubeData.dimensions.map((dim) => [ + dim.iri, + dim.__typename === "OrdinalDimension" + ? flowRight( + makeOrdinalDimensionSorter(dim), + ({ label }: { label?: string }) => label + ) + : defaultSorter, + ]) + ); const tree = makeDimensionValuesTree({ - dataCubeData: datacubeResponse.dataCubeByIri, + dataCubeData, dimensionIri, hierarchy, + sorters, }); return { data: tree, diff --git a/app/utils/sorting-values.ts b/app/utils/sorting-values.ts new file mode 100644 index 000000000..2a674060a --- /dev/null +++ b/app/utils/sorting-values.ts @@ -0,0 +1,33 @@ +import { DataCubeObservationsQuery } from "../graphql/query-hooks"; + +export const defaultSorter = ({ + value, + label, +}: { + value?: string; + label?: string; +}) => label || value; + +export const makeOrdinalDimensionSorter = ( + dimension: NonNullable< + DataCubeObservationsQuery["dataCubeByIri"] + >["dimensions"][number] +) => { + const positionsByLabel = new Map( + dimension.values.map((v) => [v.label, v.position]) + ); + return (label?: string) => (label ? positionsByLabel.get(label) ?? -1 : -1); +}; + +interface Value { + label: string; + position?: number; +} + +export const valueComparator = (locale: string) => (a: Value, b: Value) => { + if (a.position !== undefined && b.position !== undefined) { + return a.position < b.position ? -1 : 1; + } else { + return a.label.localeCompare(b.label, locale); + } +};