Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 4.x][Fixes 1115] Upload progress for GKPG based on execution requests #1125

Merged
merged 1 commit into from
Jul 29, 2022
Merged

[Backport 4.x][Fixes 1115] Upload progress for GKPG based on execution requests #1125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions geonode_mapstore_client/client/js/api/geonode/v2/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ let endpoints = {
'keywords': '/api/v2/keywords',
'regions': '/api/v2/regions',
'groups': '/api/v2/groups',
'uploads': '/api/v2/uploads'
'uploads': '/api/v2/uploads',
'status': '/api/v2/resource-service/execution-status',
'exectionRequest': '/api/v2/executionrequest'
};

const RESOURCES = 'resources';
Expand All @@ -59,6 +61,8 @@ const CATEGORIES = 'categories';
const KEYWORDS = 'keywords';
const GROUPS = 'groups';
const UPLOADS = 'uploads';
const STATUS = 'status';
const EXECUTIONREQUEST = 'exectionRequest';

function addCountToLabel(name, count) {
return `${name} (${count || 0})`;
Expand Down Expand Up @@ -772,6 +776,18 @@ export const getPendingUploads = () => {
.then(({ data }) => data?.uploads);
};

export const getPendingExecutionRequests = () => {
return axios.get(parseDevHostname(endpoints[EXECUTIONREQUEST]), {
params: {
'filter{action}': 'import',
'page': 1,
'page_size': 99999
}
})
.then(({ data }) => data?.requests)
.catch(() => null);
};

export const getProcessedUploadsById = (ids) => {
return axios.get(parseDevHostname(endpoints[UPLOADS]), {
params: {
Expand Down Expand Up @@ -831,6 +847,15 @@ export const uploadDocument = ({
.then(({ data }) => (data));
};

export const getExecutionStatus = (executionId) => {
return axios.get(`${parseDevHostname(endpoints[STATUS])}/${executionId}`)
.then(({ data }) => ({...data, id: executionId, create_date: data.created }));
};

export const deleteExecutionRequest = (executionId) => {
return axios.delete(`${parseDevHostname(endpoints[EXECUTIONREQUEST])}/${executionId}`);
};

export default {
getEndpoints,
getResources,
Expand Down Expand Up @@ -864,7 +889,10 @@ export default {
downloadResource,
getDatasets,
getPendingUploads,
getPendingExecutionRequests,
getProcessedUploadsById,
getProcessedUploadsByImportId,
uploadDocument
uploadDocument,
getExecutionStatus,
deleteExecutionRequest
};
27 changes: 17 additions & 10 deletions geonode_mapstore_client/client/js/routes/UploadDataset.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import {
getPendingUploads,
getProcessedUploadsById,
getProcessedUploadsByImportId,
uploadDataset
uploadDataset,
getPendingExecutionRequests,
deleteExecutionRequest
} from '@js/api/geonode/v2';
import axios from '@mapstore/framework/libs/ajax';
import UploadListContainer from '@js/routes/upload/UploadListContainer';
Expand Down Expand Up @@ -171,20 +173,19 @@ function UploadList({
});
}
if (successfulUploads.length > 0) {
const successfulUploadsIds = successfulUploads.map(({ data }) => data?.id);
const successfulUploadsIds = successfulUploads.filter(({ data }) => !!data?.id).map(({data}) => data?.id);
const successfulUploadsNames = successfulUploads.map(({ baseName }) => baseName);
updateWaitingUploads(omit(waitingUploads, successfulUploadsNames));
getProcessedUploadsByImportId(successfulUploadsIds)

successfulUploadsIds.length > 0 && getProcessedUploadsByImportId(successfulUploadsIds)
.then((successfulUploadProcesses) => {
onSuccess(successfulUploadProcesses);
setLoading(false);
})
.catch(() => {
setLoading(false);
});
} else {
setLoading(false);
}
setLoading(false);
})
.catch(() => {
setLoading(false);
Expand Down Expand Up @@ -239,20 +240,23 @@ function ProcessingUploadList({
updatePending.current = () => {
if (!loading) {
setLoading(true);
getPendingUploads()
axios.all([getPendingUploads(), getPendingExecutionRequests()])
.then(incomingUploads => [...incomingUploads[0], ...incomingUploads[1]])
.then((newPendingUploads) => {
if (isMounted.current) {
const failedPendingUploads = pendingUploads.filter(({ state }) => state === 'INVALID');
const newIds = newPendingUploads.map(({ id }) => id);
const pendingImports = newPendingUploads.filter(({ action, exec_id: execitionId }) => !!action && action === 'import' && !deletedIds.includes(execitionId)).map(pendingImport => ({ ...pendingImport, create_date: pendingImport.created, id: pendingImport.exec_id }));
const missingIds = pendingUploads
.filter(upload => (upload.state !== 'PROCESSED' && upload.state !== 'INVALID') && !newIds.includes(upload.id) && !deletedIds.includes(upload.id))
.filter(upload => (!!upload.state && upload.state !== 'PROCESSED' && upload.state !== 'INVALID') && !newIds.includes(upload.id) && !deletedIds.includes(upload.id))
.map(({ id }) => id);
const currentProcessed = pendingUploads.filter((upload) => upload.state === 'PROCESSED');
if (missingIds.length > 0) {
getProcessedUploadsById(missingIds)
.then((processed) => {
onChange([
...failedPendingUploads,
...pendingImports,
...processed,
...currentProcessed,
...newPendingUploads
Expand All @@ -262,6 +266,7 @@ function ProcessingUploadList({
.catch(() => {
onChange([
...failedPendingUploads,
...pendingImports,
...currentProcessed,
...newPendingUploads
]);
Expand All @@ -270,6 +275,7 @@ function ProcessingUploadList({
} else {
onChange([
...failedPendingUploads,
...pendingImports,
...currentProcessed,
...newPendingUploads
]);
Expand All @@ -285,8 +291,9 @@ function ProcessingUploadList({
}
};

function handleDelete({ id, deleteUrl }) {
axios.get(deleteUrl)
function handleDelete({ id, deleteUrl = null }) {
const deleteRequest = deleteUrl ? () => axios.get(deleteUrl) : () => deleteExecutionRequest(id);
deleteRequest()
.finally(() => {
if (isMounted.current) {
setDeletedIds((ids) => [...ids, id]);
Expand Down
21 changes: 14 additions & 7 deletions geonode_mapstore_client/client/js/routes/upload/UploadCard.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ function UploadCard({
resumeUrl,
onRemove,
error,
type
type,
status,
errorLog
}) {

const { datasetMaxUploadSize, documentMaxUploadSize, maxParallelUploads } = getConfigProp('geoNodeSettings') || {};
Expand All @@ -46,7 +48,7 @@ function UploadCard({
return (
<div className="gn-upload-card">
<div className="gn-upload-card-header">
{state === 'INVALID' ? <div className="gn-upload-card-error"><FaIcon name="exclamation"/></div> : null}
{(state === 'INVALID' || status === 'failed') ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
<div className="gn-upload-card-title">
{detailUrl
? <a
Expand All @@ -58,7 +60,7 @@ function UploadCard({
</a>
: name}
</div>
{(progress < 100 && progress > 0) ? <Spinner /> : null}
{((progress < 100 && progress > 0) || status === 'running') ? <Spinner /> : null}
{onRemove
? <Button size="xs" onClick={onRemove}>
<FaIcon name="trash"/>
Expand All @@ -78,7 +80,7 @@ function UploadCard({
<Message msgId="gnviewer.completeUpload" />
</Button>
: null}
{detailUrl
{(detailUrl || status === 'finished')
? <Button
variant="primary"
href={detailUrl}
Expand All @@ -88,8 +90,12 @@ function UploadCard({
<Message msgId="gnviewer.view" />
</Button>
: null}
{state === 'INVALID'
? <ErrorMessageWithTooltip tooltipId={<Message msgId={`gnviewer.${getUploadErrorMessageFromCode(error?.code)}`} msgParams={{ limit: getUploadErrorMessageFromCode(error?.code) === 'fileExceeds' ? maxAllowedSize : maxParallelUploads }} />} />
{(state === 'INVALID' || status === 'failed')
? <>
{!errorLog ? <ErrorMessageWithTooltip tooltipId={<Message msgId={`gnviewer.${getUploadErrorMessageFromCode(error?.code)}`} msgParams={{ limit: getUploadErrorMessageFromCode(error?.code) === 'fileExceeds' ? maxAllowedSize : maxParallelUploads }} />} />
: <ErrorMessageWithTooltip tooltip={getUploadErrorMessageFromCode(null, errorLog)} />
}
</>
: null}
</div>
</div>
Expand Down Expand Up @@ -117,7 +123,8 @@ UploadCard.defaultProps = {
name: '',
state: '',
progress: 0,
type: 'document'
type: 'document',
status: ''
};

export default UploadCard;
14 changes: 10 additions & 4 deletions geonode_mapstore_client/client/js/routes/upload/UploadListContainer.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,26 +66,32 @@ function UploadListContainer({
name,
progress = 0,
state,
status,
create_date: createDate,
detail_url: detailUrl,
resume_url: resumeUrl,
delete_url: deleteUrl,
error
error,
log,
exec_id: execId,
created
}) => {
return (error !== 'CANCELED' &&
<li
key={id}
key={id || execId}
>
<UploadCard
name={name}
state={state}
detailUrl={detailUrl}
progress={progress}
createDate={createDate}
createDate={createDate || created}
resumeUrl={resumeUrl}
onRemove={deleteUrl ? () => onDelete({ id, deleteUrl }) : null}
onRemove={deleteUrl ? () => onDelete({ id, deleteUrl }) : execId ? () => onDelete({ id: execId }) : null}
error={error}
type={resourceType}
status={status}
errorLog={log}
/>
</li>
);
Expand Down
5 changes: 4 additions & 1 deletion geonode_mapstore_client/client/js/utils/ErrorUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
* LICENSE file in the root directory of this source tree.
*/

export const getUploadErrorMessageFromCode = (code) => {
export const getUploadErrorMessageFromCode = (code, log) => {
if (log) {
return log;
}
switch (code) {
case 'upload_parallelism_limit_exceeded': {
return 'parallelLimitError';
Expand Down
26 changes: 22 additions & 4 deletions geonode_mapstore_client/client/js/utils/ResourceUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,12 +605,30 @@ export const parseUploadResponse = (upload) => {

export const processUploadResponse = (response) => {
const newResponse = response.reduce((acc, currentResponse) => {
const duplicate = acc.find((upload) => upload.id === currentResponse.id);
const duplicate = acc.find((upload) => {
if (upload.id && currentResponse.id) {
return upload.id === currentResponse.id;
} else if (upload.id && currentResponse.exec_id) {
return upload.id === currentResponse.exec_id;
} else if (upload.exec_id && currentResponse.id) {
return upload.exec_id === currentResponse.id;
}
return upload.exec_id === currentResponse.exec_id;
});
if (duplicate) {
const newAcc = acc.filter((upload) => upload.id !== duplicate.id);
return [currentResponse, ...newAcc];
const newAcc = acc.filter((upload) => {
if (upload.id && currentResponse.id) {
return upload.id !== currentResponse.id;
} else if (upload.id && currentResponse.exec_id) {
return upload.id !== currentResponse.exec_id;
} else if (upload.exec_id && currentResponse.id) {
return upload.exec_id !== currentResponse.id;
}
return upload.exec_id !== currentResponse.exec_id;
});
return [{...currentResponse, ...(!currentResponse.id && {create_date: currentResponse.created, id: currentResponse.exec_id})}, ...newAcc];
}
return [currentResponse, ...acc];
return [{...currentResponse, ...(!currentResponse.id && {create_date: currentResponse.created, id: currentResponse.exec_id})}, ...acc];
}, []);

const uploads = parseUploadResponse(newResponse);
Expand Down
19 changes: 18 additions & 1 deletion geonode_mapstore_client/client/js/utils/__tests__/ResourceUtils-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,25 @@ describe('Test Resource Utils', () => {
state: 'COMPLETE',
progress: 100,
complete: true
},
{
exec_id: 23,
name: 'test3',
created: '2022-05-13T12:24:54.042291Z',
status: 'running',
complete: false
}];

expect(processUploadResponse([...prev, ...current])).toEqual([
{
exec_id: 23,
name: 'test3',
created: '2022-05-13T12:24:54.042291Z',
status: 'running',
complete: false,
create_date: '2022-05-13T12:24:54.042291Z',
id: 23
},
{
id: 1,
name: 'test1',
Expand Down Expand Up @@ -550,7 +566,8 @@ describe('Test Resource Utils', () => {
progress: 40,
complete: false,
resume_url: 'test/upload/delete/439'
}];
}
];

expect(parseUploadResponse(uploads)).toEqual([
{
Expand Down