diff --git a/UPDATING.md b/UPDATING.md index 804b7cdd24f34..6b6a5641496b3 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -44,6 +44,7 @@ assists people when migrating to a new version. - [26462](https://github.com/apache/superset/issues/26462): Removes the Profile feature given that it's not actively maintained and not widely used. - [26377](https://github.com/apache/superset/pull/26377): Removes the deprecated Redirect API that supported short URLs used before the permalink feature. - [26329](https://github.com/apache/superset/issues/26329): Removes the deprecated `DASHBOARD_NATIVE_FILTERS` feature flag. The previous value of the feature flag was `True` and now the feature is permanently enabled. +- [25510](https://github.com/apache/superset/pull/25510): Reenforces that any newly defined Python data format (other than epoch) must adhere to the ISO 8601 standard (enforced by way of validation at the API and database level) after a previous relaxation to include slashes in addition to dashes. From now on when specifying new columns, dataset owners will need to use a SQL expression instead to convert their string columns of the form %Y/%m/%d etc. to a `DATE`, `DATETIME`, etc. type. ### Potential Downtime diff --git a/superset-frontend/src/components/Datasource/DatasourceEditor.jsx b/superset-frontend/src/components/Datasource/DatasourceEditor.jsx index e4a4d0116a098..2bc92d7c0c1dd 100644 --- a/superset-frontend/src/components/Datasource/DatasourceEditor.jsx +++ b/superset-frontend/src/components/Datasource/DatasourceEditor.jsx @@ -329,7 +329,7 @@ function ColumnCollectionTable({ control={ } /> diff --git a/superset-frontend/src/components/Datasource/DatasourceEditor.test.jsx b/superset-frontend/src/components/Datasource/DatasourceEditor.test.jsx index 9a3d8455808c9..1221d8bf4f799 100644 --- a/superset-frontend/src/components/Datasource/DatasourceEditor.test.jsx +++ b/superset-frontend/src/components/Datasource/DatasourceEditor.test.jsx @@ -96,7 +96,7 @@ describe('DatasourceEditor', () => { const inputLabel = screen.getByPlaceholderText('Label'); const inputDescription = screen.getByPlaceholderText('Description'); - const inputDtmFormat = screen.getByPlaceholderText('%Y/%m/%d'); + const inputDtmFormat = screen.getByPlaceholderText('%Y-%m-%d'); const inputCertifiedBy = screen.getByPlaceholderText('Certified by'); const inputCertDetails = screen.getByPlaceholderText( 'Certification details', diff --git a/superset/daos/dataset.py b/superset/daos/dataset.py index 0a4425dbd7a66..4647e02ce68af 100644 --- a/superset/daos/dataset.py +++ b/superset/daos/dataset.py @@ -17,12 +17,15 @@ from __future__ import annotations import logging +from datetime import datetime from typing import Any +import dateutil.parser from sqlalchemy.exc import SQLAlchemyError from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn from superset.daos.base import BaseDAO +from superset.daos.exceptions import DAOUpdateFailedError from superset.extensions import db from superset.models.core import Database from superset.models.dashboard import Dashboard @@ -150,6 +153,17 @@ def validate_metrics_uniqueness(dataset_id: int, metrics_names: list[str]) -> bo ).all() return len(dataset_query) == 0 + @staticmethod + def validate_python_date_format(dt_format: str) -> bool: + if dt_format in ("epoch_s", "epoch_ms"): + return True + try: + dt_str = datetime.now().strftime(dt_format) + dateutil.parser.isoparse(dt_str) + return True + except ValueError: + return False + @classmethod def update( cls, @@ -193,6 +207,18 @@ def update_columns( then we delete. """ + for column in property_columns: + if ( + "python_date_format" in column + and column["python_date_format"] is not None + ): + if not DatasetDAO.validate_python_date_format( + column["python_date_format"] + ): + raise DAOUpdateFailedError( + "python_date_format is an invalid date/timestamp format." + ) + if override_columns: db.session.query(TableColumn).filter( TableColumn.table_id == model.id diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index 84f0453fb4f46..45a44043a5d23 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. import json -import re +from datetime import datetime from typing import Any +from dateutil.parser import isoparse from flask_babel import lazy_gettext as _ from marshmallow import fields, pre_load, Schema, ValidationError from marshmallow.validate import Length @@ -43,26 +44,25 @@ } -def validate_python_date_format(value: str) -> None: - regex = re.compile( - r""" - ^( - epoch_s|epoch_ms| - (?P%Y([-/]%m([-/]%d)?)?)([\sT](?P