Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: Skip loading example data from configs in CI #12610

Merged
merged 3 commits into from
Feb 2, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions superset/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ def load_examples_run(
print("Loading [Birth names]")
examples.load_birth_names(only_metadata, force)

print("Loading [Tabbed dashboard]")
examples.load_tabbed_dashboard(only_metadata)

if not load_test_data:
print("Loading [Random time series data]")
examples.load_random_time_series_data(only_metadata, force)
Expand Down Expand Up @@ -164,11 +167,8 @@ def load_examples_run(
print("Loading DECK.gl demo")
examples.load_deck_dash()

print("Loading [Tabbed dashboard]")
examples.load_tabbed_dashboard(only_metadata)

# load examples that are stored as YAML config files
examples.load_from_configs(force)
examples.load_from_configs(force, load_test_data)


@with_appcontext
Expand Down
10 changes: 7 additions & 3 deletions superset/examples/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import re
from pathlib import Path
from typing import Any, Dict

Expand All @@ -24,13 +25,13 @@
YAML_EXTENSIONS = {".yaml", ".yml"}


def load_from_configs(force_data: bool = False) -> None:
contents = load_contents()
def load_from_configs(force_data: bool = False, load_test_data: bool = False) -> None:
contents = load_contents(load_test_data)
command = ImportExamplesCommand(contents, overwrite=True, force_data=force_data)
command.run()


def load_contents() -> Dict[str, Any]:
def load_contents(load_test_data: bool = False) -> Dict[str, Any]:
"""Traverse configs directory and load contents"""
root = Path("examples/configs")
resource_names = resource_listdir("superset", str(root))
Expand All @@ -39,13 +40,16 @@ def load_contents() -> Dict[str, Any]:
contents: Dict[Path, str] = {}
while queue:
path_name = queue.pop()
test_re = re.compile(r"\.test\.|metadata\.yaml$")

if resource_isdir("superset", str(path_name)):
queue.extend(
path_name / child_name
for child_name in resource_listdir("superset", str(path_name))
)
elif path_name.suffix.lower() in YAML_EXTENSIONS:
if load_test_data and test_re.search(str(path_name)) is None:
continue
contents[path_name] = (
resource_stream("superset", str(path_name)).read().decode("utf-8")
)
Expand Down