Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 4.x][Fixes 925 & 1018] The upload box should accept xml and sld files along with the dataset itself #1032

Merged
merged 3 commits into from
Jun 21, 2022
Merged

[Backport 4.x][Fixes 925 & 1018] The upload box should accept xml and sld files along with the dataset itself #1032

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
557aeaf
added xml and sld
DavidQuartz Jun 3, 2022
7339957
exclude xml and sld from main files
DavidQuartz Jun 7, 2022
f87d642
Merge branch '4.x' into xml_backport
DavidQuartz Jun 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 48 additions & 38 deletions geonode_mapstore_client/client/js/routes/UploadDataset.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import PropTypes from 'prop-types';
import omit from 'lodash/omit';
import pick from 'lodash/pick';
import merge from 'lodash/merge';
import { connect } from 'react-redux';
import { createSelector } from 'reselect';
Expand All @@ -23,44 +22,64 @@ import axios from '@mapstore/framework/libs/ajax';
import UploadListContainer from '@js/routes/upload/UploadListContainer';
import UploadContainer from '@js/routes/upload/UploadContainer';
import { getConfigProp } from '@mapstore/framework/utils/ConfigUtils';
import { parseUploadResponse, processUploadResponse } from '@js/utils/ResourceUtils';
import { parseUploadResponse, processUploadResponse, parseUploadFiles } from '@js/utils/ResourceUtils';

const supportedDatasetTypes = [
{
id: 'shp',
label: 'ESRI Shapefile',
format: 'vector',
ext: ['shp'],
requires: ['shp', 'prj', 'dbf', 'shx']
requires: ['shp', 'prj', 'dbf', 'shx'],
optional: ['xml', 'sld']
},
{
id: 'tiff',
label: 'GeoTIFF',
format: 'raster',
ext: ['tiff', 'tif'],
mimeType: ['image/tiff']
mimeType: ['image/tiff'],
optional: ['xml', 'sld']
},
{
id: 'csv',
label: 'Comma Separated Value (CSV)',
format: 'vector',
ext: ['csv'],
mimeType: ['text/csv']
mimeType: ['text/csv'],
optional: ['xml', 'sld']
},
{
id: 'zip',
label: 'Zip Archive',
format: 'archive',
ext: ['zip'],
mimeType: ['application/zip']
mimeType: ['application/zip'],
optional: ['xml', 'sld']
},
{
id: 'xml',
label: 'XML Metadata File',
format: 'metadata',
ext: ['xml'],
mimeType: ['application/json'],
needsFiles: ['shp', 'prj', 'dbf', 'shx', 'csv', 'tiff', 'zip', 'sld']
},
{
id: 'sld',
label: 'Styled Layer Descriptor (SLD)',
format: 'metadata',
ext: ['sld'],
mimeType: ['application/json'],
needsFiles: ['shp', 'prj', 'dbf', 'shx', 'csv', 'tiff', 'zip', 'xml']
}
];


const supportedExtensions = supportedDatasetTypes.map(({ ext }) => ext || []).flat();
const supportedMimeTypes = supportedDatasetTypes.map(({ mimeType }) => mimeType || []).flat();
const supportedRequiresExtensions = supportedDatasetTypes.map(({ requires }) => requires || []).flat();
const supportedLabels = supportedDatasetTypes.map(({ label }) => label ).join(', ');
const supportedLabels = supportedDatasetTypes.map(({ label }) => label).join(', ');
const supportedOptionalExtensions = supportedDatasetTypes.map(({ optional }) => optional || []).flat();

function getFileNameParts(file) {
const { name } = file;
Expand Down Expand Up @@ -95,37 +114,14 @@ function UploadList({
const [loading, setLoading] = useState(false);
const [uploadContainerProgress, setUploadContainerProgress] = useState({});

function parseUploadFiles(uploadFiles) {
return Object.keys(uploadFiles)
.reduce((acc, baseName) => {
const uploadFile = uploadFiles[baseName];
const { requires = [], ext = []} = supportedDatasetTypes.find(({ id }) => id === uploadFile.type) || {};
const cleanedFiles = pick(uploadFiles[baseName].files, [...requires, ...ext]);
const filesKeys = Object.keys(cleanedFiles);
const files = requires.length > 0
? cleanedFiles
: filesKeys.length > 1
? pick(cleanedFiles, ext[0])
: cleanedFiles;
const missingExt = requires.filter((fileExt) => !filesKeys.includes(fileExt));
return {
...acc,
[baseName]: {
...uploadFile,
mainExt: filesKeys.find(key => ext.includes(key)),
files,
missingExt
}
};
}, {});
}

function updateWaitingUploads(uploadFiles) {
const newWaitingUploads = parseUploadFiles(uploadFiles);
// prepare params for parseUploadFiles
const params = { uploadFiles, supportedDatasetTypes, supportedOptionalExtensions, supportedRequiresExtensions };
const newWaitingUploads = parseUploadFiles(params);
setWaitingUploads(newWaitingUploads);
const newReadyUploads = Object.keys(newWaitingUploads)
.reduce((acc, baseName) => {
if (newWaitingUploads[baseName]?.missingExt?.length > 0) {
if (newWaitingUploads[baseName]?.missingExt?.length > 0 || newWaitingUploads[baseName]?.addMissingFiles) {
return acc;
}
return {
Expand All @@ -141,7 +137,7 @@ function UploadList({
const { type } = file;
const { ext } = getFileNameParts(file);
return {
supported: !!(supportedMimeTypes.includes(type) || supportedExtensions.includes(ext) || supportedRequiresExtensions.includes(ext)),
supported: !!(supportedMimeTypes.includes(type) || supportedExtensions.includes(ext) || supportedRequiresExtensions.includes(ext) || supportedOptionalExtensions.includes(ext)),
file
};
});
Expand All @@ -150,11 +146,14 @@ function UploadList({
.filter(({ supported }) => supported)
.reduce((acc, { file }) => {
const { ext, baseName } = getFileNameParts(file);
const type = getDatasetFileType(file);
let type = getDatasetFileType(file);
if (!type && supportedOptionalExtensions.includes(ext)) {
type = checkedFiles.length > 1 ? acc[baseName]?.type : ext;
}
return {
...acc,
[baseName]: {
type,
type: type,
files: {
...acc[baseName]?.files,
[ext]: file
Expand All @@ -173,6 +172,12 @@ function UploadList({
setUploadContainerProgress((prevFiles) => ({ ...prevFiles, [fileName]: percentCompleted }));
};

function getValidFileExt({ files }) {
const validFile = Object.keys(files).find(key => key !== 'sld' && key !== 'xml');

return validFile;
}

function handleUploadProcess() {
if (!loading) {
setLoading(true);
Expand All @@ -181,6 +186,11 @@ function UploadList({
const readyUpload = readyUploads[baseName];
cancelTokens[baseName] = axios.CancelToken;
sources[baseName] = cancelTokens[baseName].source();

const mainExt = (readyUpload.mainExt !== 'sld' && readyUpload.mainExt !== 'xml') ? readyUpload.mainExt : getValidFileExt(readyUpload);

readyUpload.mainExt = mainExt;

return uploadDataset({
file: readyUpload.files[readyUpload.mainExt],
ext: readyUpload.mainExt,
Expand Down
10 changes: 8 additions & 2 deletions geonode_mapstore_client/client/js/routes/upload/PendingUploadCard.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ function PendingUploadCard({
progress,
size,
onAbort,
error
error,
addMissingFiles
}) {
return (
<div className="gn-upload-card">
<div className="gn-upload-card-header">
{missingExt.length > 0 ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
{(missingExt.length > 0 || addMissingFiles) ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
<div className="gn-upload-card-title">{baseName}</div>
<div>
{error ? <ErrorMessageWithTooltip tooltipId={<Message msgId="gnviewer.invalidUploadMessageErrorTooltip" />} /> : null}
Expand All @@ -53,6 +54,11 @@ function PendingUploadCard({
<Message msgId="gnviewer.missingFiles" />: {missingExt.join(', ')}
</div>
</div>}
{addMissingFiles && <div className="gn-upload-card-body">
<div className="text-danger">
<Message msgId="gnviewer.addMainFiles" />
</div>
</div>}
<div className="gn-upload-card-bottom">
<ul>
{filesExt.map(ext => {
Expand Down
3 changes: 2 additions & 1 deletion geonode_mapstore_client/client/js/routes/upload/UploadContainer.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ function UploadContainer({
{waitingUploadNames.length > 0 ? (
<ul>
{waitingUploadNames.map((baseName) => {
const { files, missingExt = [], error } = waitingUploads[baseName];
const { files, missingExt = [], error, addMissingFiles = false } = waitingUploads[baseName];
const filesExt = Object.keys(files);
const size = getSize(files, filesExt);
return (
Expand All @@ -123,6 +123,7 @@ function UploadContainer({
size={size}
onAbort={abort}
error={error}
addMissingFiles={addMissingFiles}
/>
</li>
);
Expand Down
43 changes: 39 additions & 4 deletions geonode_mapstore_client/client/js/utils/ResourceUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@ import { getConfigProp, convertFromLegacy, normalizeConfig } from '@mapstore/fra
import { parseDevHostname } from '@js/utils/APIUtils';
import { ProcessTypes, ProcessStatus } from '@js/utils/ResourceServiceUtils';
import { bboxToPolygon } from '@js/utils/CoordinatesUtils';
import uniqBy from 'lodash/uniqBy';
import orderBy from 'lodash/orderBy';
import isString from 'lodash/isString';
import isObject from 'lodash/isObject';
import { uniqBy, orderBy, isString, isObject, pick, difference } from 'lodash';
import { excludeGoogleBackground, extractTileMatrixFromSources } from '@mapstore/framework/utils/LayersUtils';

/**
Expand Down Expand Up @@ -626,3 +623,41 @@ export const cleanUrl = (targetUrl) => {
...(hash && { hash })
});
};

export const parseUploadFiles = (data) => {
const { uploadFiles = {}, supportedDatasetTypes = [], supportedOptionalExtensions = [], supportedRequiresExtensions = [] } = data;
const mainFileTypes = supportedDatasetTypes.filter(file => !file.needsFiles);
const mainFileTypeKeys = mainFileTypes.map(({ id }) => id);

return Object.keys(uploadFiles)
.reduce((acc, baseName) => {
const uploadFile = uploadFiles[baseName] || {};
const { requires = [], ext = [], optional = [], needsFiles = [] } = supportedDatasetTypes.find(({ id }) => id === uploadFile.type) || {};
const cleanedFiles = pick(uploadFile.files, [...requires, ...ext, ...optional, ...needsFiles]);
const filesKeys = Object.keys(cleanedFiles);
const files = requires.length > 0
? cleanedFiles
: filesKeys.length > 1
? pick(cleanedFiles, supportedOptionalExtensions.includes(ext[0]) ? [...needsFiles, ext[0]] : ext[0])
: cleanedFiles;
const newFileKeys = Object.keys(files);
const requiredFilesIncluded = newFileKeys.filter((id) => supportedRequiresExtensions.includes(id)) || [];
const missingExt = requires.length > 0
? requires.filter((fileExt) => !filesKeys.includes(fileExt))
: requiredFilesIncluded.length > 0 ? difference(supportedRequiresExtensions, requiredFilesIncluded) : [];

const mainExt = filesKeys.find(key => ext.includes(key));
const addMissingFiles = supportedOptionalExtensions.includes(mainExt) && missingExt?.length === 0 && !(mainFileTypeKeys.some((type) => newFileKeys.includes(type)));

return {
...acc,
[baseName]: {
...uploadFile,
mainExt,
files,
missingExt,
addMissingFiles
}
};
}, {});
};
Loading