From 28a1320da9923cf20fba6112dbb5917f11330c7b Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Mon, 16 May 2022 12:12:50 +0200 Subject: [PATCH 01/47] start with postgis DB --- README.md | 6 + .../actinia-parallel-plugin-test.cfg | 9 ++ docker/actinia.cfg | 9 ++ docker/docker-compose.yml | 16 +++ docker/postgresql_init_data/gis.sql | 66 ++++++++++ requirements.txt | 6 + setup.py | 10 +- .../api/parallel_processing.py | 1 + src/actinia_parallel_plugin/core/jobtable.py | 61 +++++++++ src/actinia_parallel_plugin/endpoints.py | 5 + .../model/jobtabelle.py | 91 +++++++++++++ .../resources/config.py | 124 ++++++++++++++++++ .../resources/logging.py | 120 +++++++++++++++++ .../resources/migrations/0000.info.py | 42 ++++++ 14 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 docker/postgresql_init_data/gis.sql create mode 100644 src/actinia_parallel_plugin/core/jobtable.py create mode 100644 src/actinia_parallel_plugin/model/jobtabelle.py create mode 100644 src/actinia_parallel_plugin/resources/config.py create mode 100644 src/actinia_parallel_plugin/resources/logging.py create mode 100644 src/actinia_parallel_plugin/resources/migrations/0000.info.py diff --git a/README.md b/README.md index af1e134..3047006 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ docker-compose -f docker/docker-compose.yml run --rm --service-ports --entrypoin gunicorn -b 0.0.0.0:8088 -w 1 --access-logfile=- -k gthread actinia_core.main:flask_app ``` +### Postgis +Connect to postgis DB from actinia-core docker container: +``` +psql -U actinia -h postgis -d gis +``` + ### Hints * If you have no `.git` folder in the plugin folder, you need to set the diff --git a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg index e41b90f..05adc95 100644 --- a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg +++ b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg @@ -22,3 +22,12 @@ tmp_workdir = /actinia_core/workspace/tmp download_cache = /actinia_core/workspace/download_cache secret_key = token_signing_key_changeme save_interim_results = False + +[JOBTABLE] +host = postgis +port = 5432 +database = gis +user = actinia +schema = actinia +table = tab_jobs +id_field = idpk_jobs diff --git a/docker/actinia.cfg b/docker/actinia.cfg index 10079cc..0baab7c 100644 --- a/docker/actinia.cfg +++ b/docker/actinia.cfg @@ -26,3 +26,12 @@ tmp_workdir = /actinia_core/workspace/tmp download_cache = /actinia_core/workspace/download_cache secret_key = token_signing_key_changeme save_interim_results = True + +[JOBTABLE] +host = postgis +port = 5432 +database = gis +user = actinia +schema = actinia +table = tab_jobs +id_field = idpk_jobs diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9fff8da..99b9664 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,8 +9,11 @@ services: - ..:/src/actinia-parallel-plugin/. ports: - "8088:8088" + environment: + - JOBTABLE_PW=actinia depends_on: - redis + - postgis cap_add: - SYS_PTRACE networks: @@ -35,6 +38,19 @@ services: networks: - actinia + postgis: + image: mdillon/postgis:9.6-alpine + # network_mode: host + environment: + POSTGRES_USER: actinia + POSTGRES_PASSWORD: actinia + volumes: + - ./postgresql_init_data:/docker-entrypoint-initdb.d + - ./postgresql_data:/var/lib/postgresql/data:Z + networks: + - actinia + + networks: actinia: ipam: diff --git a/docker/postgresql_init_data/gis.sql b/docker/postgresql_init_data/gis.sql new file mode 100644 index 0000000..2838647 --- /dev/null +++ b/docker/postgresql_init_data/gis.sql @@ -0,0 +1,66 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- TOC entry 3593 (class 1262 OID 16386) +-- Name: gis; Type: DATABASE; Schema: -; Owner: - +-- + +CREATE DATABASE gis WITH TEMPLATE = template0 ENCODING = 'UTF8' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8'; + + +\connect gis + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- TOC entry 7 (class 2615 OID 53605) +-- Name: actinia; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA actinia; + +-- +-- TOC entry 1 (class 3079 OID 12390) +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + +-- +-- TOC entry 3595 (class 0 OID 0) +-- Dependencies: 1 +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +-- +-- TOC entry 2 (class 3079 OID 34955) +-- Name: postgis; Type: EXTENSION; Schema: -; Owner: - +-- + + +SET default_tablespace = ''; + +SET default_with_oids = false; diff --git a/requirements.txt b/requirements.txt index 3ce41f4..95754a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,9 @@ # numpy==1.13.3 # scipy==1.0 # + + +python-json-logger==0.1.11 +peewee +yoyo-migrations +psycopg2 diff --git a/setup.py b/setup.py index b16e95f..d32e672 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ Learn more under: https://pyscaffold.org/ """ import sys +from pathlib import Path from pkg_resources import VersionConflict, require from setuptools import setup @@ -20,4 +21,11 @@ if __name__ == "__main__": - setup(use_pyscaffold=True) + + parent_dir = Path(__file__).resolve().parent + + setup( + use_pyscaffold=True, + install_requires=parent_dir.joinpath( + "requirements.txt").read_text().splitlines(), + ) diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index a2a6f77..d99f25f 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -46,6 +46,7 @@ def __init__(self): @swagger.doc(helloworld.describeHelloWorld_get_docs) def get(self): """Get 'Hello world!' as answer string.""" + import pdb; pdb.set_trace() return SimpleStatusCodeResponseModel(status=200, message=self.msg) # TODO start a parallel processing job as batch job diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py new file mode 100644 index 0000000..4ee1c73 --- /dev/null +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Module to communicate with jobtable +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +from datetime import datetime + +from playhouse.shortcuts import model_to_dict +from peewee import Expression, AutoField, OperationalError +from uuid import uuid4 +from yoyo import read_migrations +from yoyo import get_backend + +from actinia_parallel_plugin.model.jobtabelle import Job, jobdb +from actinia_parallel_plugin.resources.config import JOBTABLE +from actinia_parallel_plugin.resources.logging import log + + +# We used `jobdb.connect(reuse_if_open=True)` at the beginning +# of every method. Now we use `with jobdb:` as described in the +# peewee docs but we still try to jobdb.close() at the end of +# each method. + +def initJobDB(): + """Create jobtable on startup.""" + Job.create_table(safe=True) + log.debug('Created jobtable if not exists') + + +def applyMigrations(): + backend = get_backend( + 'postgres://%s:%s@%s/%s?schema=%s' % + (JOBTABLE.user, JOBTABLE.pw, JOBTABLE.host, JOBTABLE.database, + JOBTABLE.schema)) + migrations = read_migrations('actinia_parallel_plugin/resources/migrations') + + with backend.lock(): + backend.apply_migrations(backend.to_apply(migrations)) + log.debug('Applied migrations.') diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index ed2c485..17e4685 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -27,6 +27,7 @@ from actinia_parallel_plugin.api.parallel_processing import \ ParallelProcessingResource +from actinia_parallel_plugin.core.jobtable import initJobDB, applyMigrations # endpoints loaded if run as actinia-core plugin @@ -35,3 +36,7 @@ def create_endpoints(flask_api): apidoc = flask_api apidoc.add_resource(ParallelProcessingResource, "/processing_parallel") + + # initilalize jobtable + initJobDB() + applyMigrations() diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py new file mode 100644 index 0000000..ebd690e --- /dev/null +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Response models +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +from peewee import Model, CharField, DateTimeField, AutoField, IntegerField +from playhouse.postgres_ext import BinaryJSONField +from playhouse.pool import PooledPostgresqlExtDatabase + +from actinia_parallel_plugin.resources.config import JOBTABLE +from actinia_parallel_plugin.resources.logging import log + + +log.debug("Database config loaded: " + JOBTABLE.host + ":" + JOBTABLE.port + + "/" + JOBTABLE.database + "/" + + JOBTABLE.schema + "." + JOBTABLE.table) + + +"""database connection""" + +jobdb = PooledPostgresqlExtDatabase( + JOBTABLE.database, **{ + 'host': JOBTABLE.host, + 'port': JOBTABLE.port, + 'user': JOBTABLE.user, + 'password': JOBTABLE.pw, + 'max_connections': 8, + 'stale_timeout': 300 + } +) + + +class BaseModel(Model): + """Base Model for tables in jobdb + """ + class Meta: + database = jobdb + + +class Job(BaseModel): + """Model for jobtable in database + """ + idpk_jobs = AutoField() + process = CharField(null=True) + feature_type = CharField(null=True) + rule_configuration = BinaryJSONField(null=True) + job_description = BinaryJSONField(null=True) + time_created = DateTimeField(null=True) + time_started = DateTimeField(null=True) + time_estimated = DateTimeField(null=True) + time_ended = DateTimeField(null=True) + metadata = CharField(null=True) + status = CharField(null=True) + actinia_core_response = BinaryJSONField(null=True) + actinia_core_jobid = CharField(null=True) + actinia_core_url = CharField(null=True) + actinia_core_platform = CharField(null=True) + actinia_core_platform_name = CharField(null=True) + terraformer_id = IntegerField(null=True) + terraformer_response = BinaryJSONField(null=True) + creation_uuid = CharField(null=True) + message = CharField(null=True) + # add a potential parent_job + batch_id = IntegerField(null=True) + processing_block = IntegerField(null=True) + batch_description = BinaryJSONField(null=True) + + class Meta: + table_name = JOBTABLE.table + schema = JOBTABLE.schema diff --git a/src/actinia_parallel_plugin/resources/config.py b/src/actinia_parallel_plugin/resources/config.py new file mode 100644 index 0000000..bc77804 --- /dev/null +++ b/src/actinia_parallel_plugin/resources/config.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Configuration file +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import configparser +import os +from pathlib import Path + + +# # config can be overwritten by mounting *.ini files into folders inside +# # the config folder. +# DEFAULT_CONFIG_PATH = "config" +# CONFIG_FILES = [str(f) for f in Path( +# DEFAULT_CONFIG_PATH).glob('**/*.ini') if f.is_file()] +# GENERATED_CONFIG = DEFAULT_CONFIG_PATH + '/actinia-parallel-plugin.cfg' + +DEFAULT_CONFIG_PATH = os.getenv('DEFAULT_CONFIG_PATH', "/etc/default/actinia") +GENERATED_CONFIG = os.path.join( + os.path.dirname(DEFAULT_CONFIG_PATH), 'actinia-parallel-plugin.cfg') +if not os.path.isfile(DEFAULT_CONFIG_PATH): + open(DEFAULT_CONFIG_PATH, 'a').close() +CONFIG_FILES = [DEFAULT_CONFIG_PATH] + + +class JOBTABLE: + """Default config for database connection for jobtabelle + """ + host = 'localhost' + port = '5432' + database = 'gis' + user = 'gis' + pw = 'gis' + schema = 'actinia' + table = 'tab_jobs' + id_field = 'idpk_jobs' + batch_id_field = "batch_id" + resource_id_field = "actinia_core_jobid" + + +class LOGCONFIG: + """Default config for logging + """ + logfile = 'actinia-gdi.log' + level = 'DEBUG' + type = 'stdout' + + +class Configfile: + + def __init__(self): + """ + This class will overwrite the config classes above when config files + named DEFAULT_CONFIG_PATH/**/*.ini exist. + On first import of the module it is initialized. + """ + + config = configparser.ConfigParser() + config.read(CONFIG_FILES) + + if len(config) <= 1: + # print("Could not find any config file, using default values.") + return + + with open(GENERATED_CONFIG, 'w') as configfile: + config.write(configfile) + + # JOBTABLE + if config.has_section("JOBTABLE"): + if config.has_option("JOBTABLE", "host"): + JOBTABLE.host = config.get("JOBTABLE", "host") + if config.has_option("JOBTABLE", "port"): + JOBTABLE.port = config.get("JOBTABLE", "port") + if config.has_option("JOBTABLE", "database"): + JOBTABLE.database = config.get("JOBTABLE", "database") + if config.has_option("JOBTABLE", "user"): + JOBTABLE.user = config.get("JOBTABLE", "user") + if config.has_option("JOBTABLE", "pw"): + JOBTABLE.pw = config.get("JOBTABLE", "pw") + if config.has_option("JOBTABLE", "schema"): + JOBTABLE.schema = config.get("JOBTABLE", "schema") + if config.has_option("JOBTABLE", "table"): + JOBTABLE.table = config.get("JOBTABLE", "table") + if config.has_option("JOBTABLE", "id_field"): + JOBTABLE.id_field = config.get("JOBTABLE", "id_field") + + # overwrite values if ENV values exist: + if os.environ.get('JOBTABLE_USER'): + JOBTABLE.user = os.environ['JOBTABLE_USER'] + if os.environ.get('JOBTABLE_PW'): + JOBTABLE.pw = os.environ['JOBTABLE_PW'] + + # LOGGING + if config.has_section("LOGCONFIG"): + if config.has_option("LOGCONFIG", "logfile"): + LOGCONFIG.logfile = config.get("LOGCONFIG", "logfile") + if config.has_option("LOGCONFIG", "level"): + LOGCONFIG.level = config.get("LOGCONFIG", "level") + if config.has_option("LOGCONFIG", "type"): + LOGCONFIG.type = config.get("LOGCONFIG", "type") + + +init = Configfile() diff --git a/src/actinia_parallel_plugin/resources/logging.py b/src/actinia_parallel_plugin/resources/logging.py new file mode 100644 index 0000000..f5f3e33 --- /dev/null +++ b/src/actinia_parallel_plugin/resources/logging.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The logger +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import logging +from datetime import datetime +from logging import FileHandler + +from colorlog import ColoredFormatter +from pythonjsonlogger import jsonlogger + +from actinia_parallel_plugin.resources.config import LOGCONFIG + + +# Notice: do not call logging.warning (will create new logger for ever) +# logging.warning("called actinia_gdi logger after 1") + +log = logging.getLogger('actinia-parallel-plugin') +werkzeugLog = logging.getLogger('werkzeug') +gunicornLog = logging.getLogger('gunicorn') + + +def setLogFormat(veto=None): + logformat = "" + if LOGCONFIG.type == 'json' and not veto: + logformat = CustomJsonFormatter('%(time) %(level) %(component) %(module)' + '%(message) %(pathname) %(lineno)' + '%(processName) %(threadName)') + else: + logformat = ColoredFormatter( + '%(log_color)s[%(asctime)s] %(levelname)-10s: %(name)s.%(module)-' + '10s -%(message)s [in %(pathname)s:%(lineno)d]%(reset)s' + ) + return logformat + + +def setLogHandler(logger, type, format): + if type == 'stdout': + handler = logging.StreamHandler() + elif type == 'file': + # For readability, json is never written to file + handler = FileHandler(LOGCONFIG.logfile) + + handler.setFormatter(format) + logger.addHandler(handler) + + +class CustomJsonFormatter(jsonlogger.JsonFormatter): + def add_fields(self, log_record, record, message_dict): + super(CustomJsonFormatter, self).add_fields( + log_record, record, message_dict) + + # (Pdb) dir(record) + # ... 'args', 'created', 'exc_info', 'exc_text', 'filename', 'funcName' + # ,'getMessage', 'levelname', 'levelno', 'lineno', 'message', 'module', + # 'msecs', 'msg', 'name', 'pathname', 'process', 'processName', + # 'relativeCreated', 'stack_info', 'thread', 'threadName'] + + now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') + log_record['time'] = now + log_record['level'] = record.levelname + log_record['component'] = record.name + + +def createLogger(): + # create logger, set level and define format + log.setLevel(getattr(logging, LOGCONFIG.level)) + fileformat = setLogFormat('veto') + stdoutformat = setLogFormat() + setLogHandler(log, 'file', fileformat) + setLogHandler(log, 'stdout', stdoutformat) + + +def createWerkzeugLogger(): + werkzeugLog.setLevel(getattr(logging, LOGCONFIG.level)) + fileformat = setLogFormat('veto') + stdoutformat = setLogFormat() + setLogHandler(werkzeugLog, 'file', fileformat) + setLogHandler(werkzeugLog, 'stdout', stdoutformat) + + +def createGunicornLogger(): + gunicornLog.setLevel(getattr(logging, LOGCONFIG.level)) + fileformat = setLogFormat('veto') + stdoutformat = setLogFormat() + setLogHandler(gunicornLog, 'file', fileformat) + setLogHandler(gunicornLog, 'stdout', stdoutformat) + # gunicorn already has a lot of children logger, e.g gunicorn.http, + # gunicorn.access. These lines deactivate their default handlers. + for name in logging.root.manager.loggerDict: + if "gunicorn." in name: + logging.getLogger(name).propagate = True + logging.getLogger(name).handlers = [] + + +createLogger() +createWerkzeugLogger() +createGunicornLogger() diff --git a/src/actinia_parallel_plugin/resources/migrations/0000.info.py b/src/actinia_parallel_plugin/resources/migrations/0000.info.py new file mode 100644 index 0000000..a5395ca --- /dev/null +++ b/src/actinia_parallel_plugin/resources/migrations/0000.info.py @@ -0,0 +1,42 @@ +''' +The initial table is created with peewee (initInstanceDB). +All further adjustments should be made with yoyo-migrations. +For a new adjustment, a new file is needed, stored next to this one. +Yoyo stores already applied migrations in the db: + +gis=# select * from _yoyo_migration ; +migration_hash | migration_id | + applied_at_utc +------------------------------------------------------------------+----------------------+---------------------------- +b9faf3aa8fe158938471e8275bf6f7dc6d49bd4c5e7a89953de4b790b711eba8 | + 0001.add-status_info | 2020-06-10 15:13:45.250212 + +gis=# select * from _yoyo_log ; +id | + migration_hash | migration_id | operation | + username | hostname | comment | created_at_utc +--------------------------------------+------------------------------------------------------------------+----------------------+-----------+----------+----------+---------+--------------------------- +fa2e5dd8-ab2c-11ea-9e24-6057186705c0 | + b9faf3aa8fe158938471e8275bf6f7dc6d49bd4c5e7a89953de4b790b711eba8 | + 0001.add-status_info | apply | default | carmen | | 2020-06-10 + 15:13:45.24777 +(1 row) + +Rollbacks are also possible but currently not integrated in our code. We should +use them, if we have more comlex migration scripts which should not be applied +only partwise. +Pure SQL scripts are also possible but cannot replace variables as needed here. + +For more information, see https://ollycope.com/software/yoyo/latest/ + +''' + +from yoyo import step +from actinia_parallel_plugin.resources.config import JOBTABLE + +# dummy migration to test functionality +steps = [ + step( + "select * from %s limit 1" % JOBTABLE.table + ) +] From 03b7800115677ffeb3e51fbfb50d61eaf6714ffc Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 18 May 2022 16:07:31 +0200 Subject: [PATCH 02/47] zwischenstand --- src/actinia_parallel_plugin/api/batch.py | 77 +++ .../api/parallel_processing.py | 179 ++++++- src/actinia_parallel_plugin/apidocs/batch.py | 71 +++ .../apidocs/examples/__init__.py | 0 .../batchjob_post_response_example.json | 302 ++++++++++++ src/actinia_parallel_plugin/core/batches.py | 326 +++++++++++++ src/actinia_parallel_plugin/core/jobs.py | 440 ++++++++++++++++++ src/actinia_parallel_plugin/core/jobtable.py | 355 ++++++++++++++ .../core/persistent_processing.py | 212 +++++++++ src/actinia_parallel_plugin/endpoints.py | 20 +- src/actinia_parallel_plugin/model/batch.py | 269 +++++++++++ .../model/batch_process_chain.py | 129 +++++ test_postbodies/parallel_processing.json | 63 +++ test_postbodies/parallel_processing_soll.json | 138 ++++++ 14 files changed, 2560 insertions(+), 21 deletions(-) create mode 100644 src/actinia_parallel_plugin/api/batch.py create mode 100644 src/actinia_parallel_plugin/apidocs/batch.py create mode 100644 src/actinia_parallel_plugin/apidocs/examples/__init__.py create mode 100644 src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json create mode 100644 src/actinia_parallel_plugin/core/batches.py create mode 100644 src/actinia_parallel_plugin/core/jobs.py create mode 100644 src/actinia_parallel_plugin/core/persistent_processing.py create mode 100644 src/actinia_parallel_plugin/model/batch.py create mode 100644 src/actinia_parallel_plugin/model/batch_process_chain.py create mode 100644 test_postbodies/parallel_processing.json create mode 100644 test_postbodies/parallel_processing_soll.json diff --git a/src/actinia_parallel_plugin/api/batch.py b/src/actinia_parallel_plugin/api/batch.py new file mode 100644 index 0000000..5ffc306 --- /dev/null +++ b/src/actinia_parallel_plugin/api/batch.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2021-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Module to communicate with jobtable +""" + +__license__ = "GPLv3" +__author__ = "Julia Haas, Guido Riembauer, Anika Weinmann" +__copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +from flask_restful import Resource +from flask_restful_swagger_2 import swagger +from flask import make_response, jsonify, request + +from actinia_core.models.response_models import \ + SimpleResponseModel + +from actinia_parallel_plugin.resources.logging import log +from actinia_parallel_plugin.core.batches import ( + createBatchResponseDict, + getJobsByBatchId, +) +from actinia_parallel_plugin.apidocs import batch + + +class BatchJobsId(Resource): + """ Definition for endpoint + @app.route('processing_parallel/batchjobs/') + + Contains HTTP GET endpoint + Contains HTTP POST endpoint + Contains swagger documentation + """ + @swagger.doc(batch.batchjobId_get_docs) + def get(self, batchid): + if batchid is None: + return make_response("No batchid was given", 400) + + log.info(("\n Received HTTP GET request for batch" + f" with id {str(batchid)}")) + + jobs = getJobsByBatchId(batch) + if len(jobs) == 0: + res = (jsonify(SimpleResponseModel( + status=404, + message='Either batchid does not exist or there was a ' + 'connection problem to the database. Please ' + 'try again later.' + ))) + return make_response(res, 404) + else: + resp_dict = createBatchResponseDict(jobs) + return make_response(jsonify(resp_dict), 200) + + # no docs because 405 + def post(self, batchid): + res = jsonify(SimpleResponseModel( + status=405, + message="Method Not Allowed" + )) + return make_response(res, 405) diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index d99f25f..6d8dfeb 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -24,40 +24,181 @@ __copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" - -from flask import request, make_response +import json +import pickle +from flask import request, make_response, jsonify from flask_restful_swagger_2 import swagger -from flask_restful_swagger_2 import Resource +# from flask_restful_swagger_2 import Resource + +from actinia_core.models.response_models import \ + SimpleResponseModel +from actinia_core.rest.base.resource_base import ResourceBase +from actinia_core.core.common.redis_interface import enqueue_job from actinia_parallel_plugin.apidocs import helloworld +from actinia_parallel_plugin.core.batches import ( + createBatch, + createBatchId, + createBatchResponseDict, + getJobsByBatchId, + # startProcessingBlock, +) +from actinia_parallel_plugin.core.jobtable import ( + getJobById, +) from actinia_parallel_plugin.model.response_models import ( SimpleStatusCodeResponseModel, ) -from actinia_parallel_plugin.core.example import transform_input +from actinia_parallel_plugin.model.batch_process_chain import ( + # BatchProcessChain, + SingleJob, +) +from actinia_parallel_plugin.core.jobtable import updateJobByID +from actinia_parallel_plugin.core.jobs import updateJob +from actinia_parallel_plugin.resources.logging import log +from actinia_parallel_plugin.core.persistent_processing import start_job -class ParallelProcessingResource(Resource): +class AsyncParallelPersistentResource(ResourceBase): """Resource for parallel processing""" def __init__(self): - self.msg = "Hello world!" + super(AsyncParallelPersistentResource, self).__init__() + self.location_name = None + self.mapset_name = None + self.batch_id = None - # TODO get all batch jobs @swagger.doc(helloworld.describeHelloWorld_get_docs) - def get(self): + def get(self, location_name, mapset_name): """Get 'Hello world!' as answer string.""" - import pdb; pdb.set_trace() return SimpleStatusCodeResponseModel(status=200, message=self.msg) - # TODO start a parallel processing job as batch job - @swagger.doc(helloworld.describeHelloWorld_post_docs) - def post(self): - """Hello World post method with name from postbody.""" + # def prepare_actinia(self): + # e.g. start a VM and check connection to actinia-core on it + # return things + + def _start_job(self, process, process_chain, jobid): + """Starting job in running actinia-core instance and update job db.""" + job, err = getJobById(jobid) + # TODO prepare_actinia ? + # TODO execute_actinia ? + # TODO goodby_actinia ? +# has_json = False +# self.request_data = pc + + rdc = self.preprocess( + has_json=True, + location_name=self.location_name, + mapset_name=self.mapset_name + ) + if rdc: + block = 1 + from actinia_parallel_plugin.core.persistent_processing import \ + ParallelPersistentProcessing + processing = ParallelPersistentProcessing( + rdc, self.batch_id, block, jobid) + processing.run(process_chain) + + # enqueue_job( + # self.job_timeout, + # start_job, + # rdc, + # self.batch_id, + # block, + # jobid, + # json.dumps(process_chain) + # ) - req_data = request.get_json(force=True) - if isinstance(req_data, dict) is False or "name" not in req_data: - return make_response("Missing name in JSON content", 400) - name = req_data["name"] - msg = transform_input(name) + # html_code, response_model = pickle.loads(self.response_data) + # job = updateJob(resourceId, actiniaCoreResp, jobid) + # job = updateJobByID( + # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId + # ) + job = getJobById(jobid)[0] + return job + # return make_response(jsonify(response_model), html_code) + + # initial actinia update, therefore with resourceId + # job = updateJobWithActiniaByID( + # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId + # ) + # return job + + def _start_processing_block(self, jobs, block): + """Starts first processing block of jobs from batch process. + """ + jobs_to_start = [ + job for job in jobs if job["processing_block"] == block] + jobs_responses = [] + for job in jobs_to_start: + process_chain = dict() + process_chain["list"] = job["rule_configuration"]["list"] + process_chain["version"] = job["rule_configuration"]["version"] + start_kwargs = { + "process": job["process"], + # "pc": SingleJob(**job["job_description"]), + "process_chain": process_chain, + "jobid": job["idpk_jobs"], + # "actinia_core_platform": job["actinia_core_platform"], + # "actinia_core_url": job["actinia_core_url"] + } + job_entry = self._start_job(**start_kwargs) + jobs_responses.append(job_entry) + return jobs_responses + + # TODO get all batch jobs + @swagger.doc(helloworld.describeHelloWorld_get_docs) + # def get(self): + def post(self, location_name, mapset_name): + """Persistent parallel processing.""" + + self.location_name = location_name + self.mapset_name = mapset_name + + json_dict = request.get_json(force=True) + log.info("Received HTTP POST with batchjob: %s" % + str(json_dict)) + + # assign new batchid + self.batch_id = createBatchId() + # create processing blocks and insert jobs into jobtable + jobs_in_db = createBatch(json_dict, "persistent", self.batch_id) + if jobs_in_db is None: + res = (jsonify(SimpleResponseModel( + status=500, + message=('Error: Batch Processing Chain JSON has no ' + 'jobs.') + ))) + return make_response(res, 500) + + # start first processing block + first_jobs = self._start_processing_block(jobs_in_db, 1) + import pdb; pdb.set_trace() + first_status = [entry["status"] for entry in first_jobs] + all_jobs = getJobsByBatchId(self.batch_id, "persistent") + if None in first_jobs: + res = (jsonify(SimpleResponseModel( + status=500, + message=('Error: There was a problem starting the ' + 'first jobs of the batchjob.') + ))) + return make_response(res, 500) + elif "ERROR" not in first_status: + return make_response(jsonify(createBatchResponseDict(all_jobs)), + 201) + else: + return make_response(jsonify(createBatchResponseDict(all_jobs)), + 412) - return SimpleStatusCodeResponseModel(status=200, message=msg) + # # TODO start a parallel processing job as batch job + # @swagger.doc(helloworld.describeHelloWorld_post_docs) + # def post(self): + # """Hello World post method with name from postbody.""" + # + # req_data = request.get_json(force=True) + # if isinstance(req_data, dict) is False or "name" not in req_data: + # return make_response("Missing name in JSON content", 400) + # name = req_data["name"] + # msg = transform_input(name) + # + # return SimpleStatusCodeResponseModel(status=200, message=msg) diff --git a/src/actinia_parallel_plugin/apidocs/batch.py b/src/actinia_parallel_plugin/apidocs/batch.py new file mode 100644 index 0000000..2122f7d --- /dev/null +++ b/src/actinia_parallel_plugin/apidocs/batch.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2021-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Documentation objects for batch endpoints +""" + +__license__ = "GPLv3" +__author__ = "Guido Riembauer, Anika Weinmann" +__copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +from actinia_core.models.response_models import \ + SimpleResponseModel + +from actinia_parallel_plugin.model.response_models import ( + SimpleStatusCodeResponseModel, +) +from actinia_parallel_plugin.model.batch import ( + BatchJobResponseModel, +) + + +batchjobId_get_docs = { + "summary": "Returns batchjob by batchid.", + "description": ("This request will get the summary for the requested " + "batchjob and all corresponding jobs from the jobtable."), + "tags": [ + "processing" + ], + "parameters": [ + { + "in": "path", + "name": "batchid", + "type": "string", + "description": "a batchid", + "required": True + } + ], + "responses": { + "200": { + "description": ("The batchjob summary of the requested batchjob " + "and all corresponding jobs"), + "schema": BatchJobResponseModel + }, + "400": { + "description": ("A short error message in case no batchid was " + "provided") + }, + "404": { + "description": ("An error message in case the batchid was " + "not found"), + "schema": SimpleResponseModel + } + } +} diff --git a/src/actinia_parallel_plugin/apidocs/examples/__init__.py b/src/actinia_parallel_plugin/apidocs/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json new file mode 100644 index 0000000..4aeb485 --- /dev/null +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -0,0 +1,302 @@ +{ + "actinia_core_jobid": [ + "None", + "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", + "resource_id-7565f55f-b798-417e-8441-e148df6194d4" + ], + "actinia_core_platform": "TODO", + "actinia_core_platform_name": "example_name", + "actinia_core_response": { + "resource_id-7565f55f-b798-417e-8441-e148df6194d4": { + "accept_datetime": "2021-11-26 09:52:54.961197", + "accept_timestamp": 1637920374.9611967, + "api_info": { + "endpoint": "asyncephemeralexportresource", + "method": "POST", + "path": "/api/v1/locations/utm32n/processing_async_export", + "request_url": "http://actinia-core-docker:8088/api/v1/locations/utm32n/processing_async_export" + }, + "datetime": "2021-11-26 09:52:54.962316", + "http_code": 200, + "message": "Resource accepted", + "process_chain_list": [], + "process_results": {}, + "resource_id": "resource_id-7565f55f-b798-417e-8441-e148df6194d4", + "status": "accepted", + "time_delta": 0.001125335693359375, + "timestamp": 1637920374.9623156, + "urls": { + "resources": [], + "status": "http://actinia-core-docker:8088/api/v1/resources/actinia-gdi/resource_id-7565f55f-b798-417e-8441-e148df6194d4" + }, + "user_id": "actinia-gdi" + }, + "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2": { + "accept_datetime": "2021-11-26 09:52:54.610856", + "accept_timestamp": 1637920374.6108556, + "api_info": { + "endpoint": "asyncephemeralexportresource", + "method": "POST", + "path": "/api/v1/locations/utm32n/processing_async_export", + "request_url": "http://actinia-core-docker:8088/api/v1/locations/utm32n/processing_async_export" + }, + "datetime": "2021-11-26 09:52:54.782627", + "http_code": 200, + "message": "Processing successfully finished", + "process_chain_list": [ + { + "list": [ + { + "flags": "p", + "id": "g_region_test_0", + "inputs": [ + { + "param": "w", + "value": "366166" + }, + { + "param": "s", + "value": "5628872" + }, + { + "param": "e", + "value": "409135" + }, + { + "param": "n", + "value": "5681310" + }, + { + "param": "res", + "value": "10" + } + ], + "module": "g.region" + } + ], + "version": "1", + "webhooks": { + "finished": "http://actinia-gdi:5000/resources/processes/operations/update", + "update": "http://actinia-gdi:5000/resources/processes/operations/update" + } + } + ], + "process_log": [ + { + "executable": "g.region", + "id": "g_region_test_0", + "mapset_size": 429, + "parameter": [ + "w=366166", + "s=5628872", + "e=409135", + "n=5681310", + "res=10", + "-p" + ], + "return_code": 0, + "run_time": 0.05014228820800781, + "stderr": [ + "" + ], + "stdout": "projection: 1 (UTM)\nzone: 32\ndatum: etrs89\nellipsoid: grs80\nnorth: 5681310\nsouth: 5628872\nwest: 366166\neast: 409135\nnsres: 9.99961861\newres: 9.99976728\nrows: 5244\ncols: 4297\ncells: 22533468\n" + } + ], + "process_results": {}, + "progress": { + "num_of_steps": 1, + "step": 1 + }, + "resource_id": "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", + "status": "finished", + "time_delta": 0.17178988456726074, + "timestamp": 1637920374.7826219, + "urls": { + "resources": [], + "status": "http://actinia-core-docker:8088/api/v1/resources/actinia-gdi/resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2" + }, + "user_id": "actinia-gdi" + } + }, + "actinia_core_url": "http://actinia-core-docker:8088/", + "actinia_gdi_batchid": 1, + "batch_description": { + "jobs": [ + { + "list": [ + { + "flags": "p", + "id": "g_region_test_0", + "inputs": [ + { + "param": "w", + "value": "366166" + }, + { + "param": "s", + "value": "5628872" + }, + { + "param": "e", + "value": "409135" + }, + { + "param": "n", + "value": "5681310" + }, + { + "param": "res", + "value": "10" + } + ], + "module": "g.region" + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "id": "g_list_0", + "inputs": [ + { + "param": "mapset", + "value": "." + }, + { + "param": "type", + "value": "raster" + } + ], + "module": "g.list" + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_test_1", + "inputs": [ + { + "param": "w", + "value": "366166" + }, + { + "param": "s", + "value": "5628872" + }, + { + "param": "e", + "value": "409135" + }, + { + "param": "n", + "value": "5681310" + }, + { + "param": "res", + "value": "10" + } + ], + "module": "g.region" + }, + { + "id": "r_mapcalc_0", + "inputs": [ + { + "param": "expression", + "value": "test_raster2 = 1" + } + ], + "module": "r.mapcalc" + }, + { + "id": "r_info_0", + "inputs": [ + { + "param": "map", + "value": "test_raster2" + } + ], + "module": "r.info", + "stdout": { + "delimiter": ",", + "format": "table", + "id": "TEST" + } + } + ], + "parallel": "false", + "version": "1" + } + ], + "processing_host": "http://actinia-core-docker:8088/", + "processing_platform_name": "example_name" + }, + "status": "RUNNING", + "creation_uuids": [ + "22638e4f-2e2e-4fd7-8221-cc5012993d2a", + "10153224-0945-4736-808e-90785f889686", + "7b31cb7d-f956-4f7a-8c16-c3eef34c72c9" + ], + "idpk_jobs": [ + "652", + "650", + "651" + ], + "process": "netdefinition", + "jobs_status": [ + { + "actinia_core_jobid": "None", + "idpk_jobs": 652, + "status": "PREPARING" + }, + { + "actinia_core_jobid": "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", + "idpk_jobs": 650, + "status": "SUCCESS" + }, + { + "actinia_core_jobid": "resource_id-7565f55f-b798-417e-8441-e148df6194d4", + "idpk_jobs": 651, + "status": "PENDING" + } + ], + "summary": { + "blocks": [ + { + "accepted": 1, + "block_num": 1, + "error": 0, + "finished": 1, + "parallel": 2, + "preparing": 0, + "running": 0, + "terminated": 0 + }, + { + "accepted": 0, + "block_num": 2, + "error": 0, + "finished": 0, + "parallel": 1, + "preparing": 1, + "running": 0, + "terminated": 0 + } + ], + "status": { + "accepted": 1, + "error": 0, + "finished": 1, + "preparing": 1, + "running": 0, + "terminated": 0 + }, + "total": 3 + } +} diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py new file mode 100644 index 0000000..8c6420e --- /dev/null +++ b/src/actinia_parallel_plugin/core/batches.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2021-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Module to communicate with jobtable +""" + +__license__ = "GPLv3" +__author__ = "Guido Riembauer, Anika Weinmann" +__copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +from actinia_parallel_plugin.core.jobtable import getAllIds +from actinia_parallel_plugin.model.batch_process_chain import ( + BatchProcessChain, + SingleJob, +) +# from actinia_gdi.core.jobtable import getAllIds, getAllJobs +from actinia_parallel_plugin.resources.logging import log +from actinia_parallel_plugin.core.jobs import insertJob +from actinia_parallel_plugin.core.jobtable import getAllJobs +# from actinia_gdi.resources.config import JOBTABLE +# from actinia_parallel_plugin.core.jobs import startJobByActiniaType +# from actinia_gdi.core.jobs import startJobByActiniaType, cancelJob + +from json import loads + + +def assignProcessingBlocks(jsonDict): + """ Function to parse input BPC and split up the joblist according + to the parallel parameter into processing blocks + """ + bpc_dict = checkBatchProcessChain(jsonDict) + if bpc_dict is None: + return None + else: + # find out individual jobs + jobs = [job for job in bpc_dict["jobs"]] + parallel_jobs = [loads(job["parallel"]) for job in jobs] + # a single parallel job makes no sense so this is corrected here + parallel_jobs_corrected = [] + for idx, job in enumerate(parallel_jobs): + if job is True: + if idx != 0 and idx != len(parallel_jobs)-1: + if (parallel_jobs[idx-1] is False + and parallel_jobs[idx+1] is False): + job = False + jobs[idx]["parallel"] = "false" + elif idx == 0: + if parallel_jobs[idx+1] is False: + job = False + jobs[idx]["parallel"] = "false" + elif idx == len(parallel_jobs)-1: + if parallel_jobs[idx-1] is False: + job = False + jobs[idx]["parallel"] = "false" + parallel_jobs_corrected.append(job) + # determine the different "processing blocks" + block_num = 1 + result_jobs = [] + for idx, job in enumerate(jobs): + parallel = parallel_jobs_corrected[idx] + prev_parallel = parallel_jobs_corrected[idx-1] + if idx > 0 and (parallel is False or prev_parallel is False): + block_num += 1 + job["processing_block"] = block_num + result_jobs.append(job) + return result_jobs + + +# def cancelBatch(batchid, process): +# """ Function to cancel all jobs that are RUNNING or PENDING +# by batchid and process +# """ +# jobs = getJobsByBatchId(batchid, process) +# cancel_jobs = [] +# id_field = JOBTABLE.id_field +# for job in jobs: +# cancel_job = cancelJob(job[id_field]) +# cancel_jobs.append(cancel_job) +# if None not in cancel_jobs: +# # then all jobs already have a resource id etc. +# return cancel_jobs +# else: +# # then there are jobs that have not been posted to actinia yet, +# # where cancelJob returns None only +# return getJobsByBatchId(batchid, process) + + +def checkBatchProcessChain(jsonDict): + """ Function to test the creation of a BatchProcessChain object from the + input JSON and return + """ + + bpc = BatchProcessChain(**jsonDict) + bpc.feature_type = "default" + # check consistency + bpc_dict = bpc.to_struct() + if len(bpc_dict["jobs"]) == 0: + log.error('Batch Processing Chain JSON has no jobs!') + return None + return bpc_dict + + +def checkProcessingBlockFinished(jobs, block): + """ Function to check if a certain processing block has finished from + an input list of jobs (db entries) + """ + status_list = [job["status"] for job + in jobs if job["processing_block"] == block] + finished = all(status == "SUCCESS" for status in status_list) + return finished + + +def createBatch(jsonDict, process, batchid): + """ Function to insert all jobs from a batch into the joblist + """ + jobs = assignProcessingBlocks(jsonDict) + if jobs is None: + return None + else: + jobs_in_db = [] + for job in jobs: + job["batch_description"] = jsonDict + # job["processing_host"] = jsonDict["processing_host"] + # job["processing_platform_name"] = jsonDict[ + # "processing_platform_name"] + job["batch_id"] = batchid + # assign the model + process_chain = SingleJob(**job) + # might be needed (?) + process_chain.feature_type = "null" + job_in_db = insertJob(job, process, process_chain) + jobs_in_db.append(job_in_db) + return jobs_in_db + + +def createBatchId(): + """ Function to create a unique BatchId + """ + existing_batch_ids = getAllBatchIds() + if len(existing_batch_ids) == 0: + batch_id = 1 + else: + batch_id = max(existing_batch_ids) + 1 + return batch_id + + +def createBatchResponseDict(jobs_list): + """ Function to create a status response dictionary from an input list of + jobs (db entries) + """ + + # get relevant information for each job + if len(jobs_list) == 0: + return {} + + # sort the jobs according to their id + jobs = sorted(jobs_list, key=lambda d: d["idpk_jobs"]) + process = jobs[0]["process"] + batch_id = jobs[0]["batch_id"] + resource_ids = [] + uuids = [] + jobs_status = [] + responses = {} + job_ids = [] + blocks = [] + for job in jobs: + resource_id = job["actinia_core_jobid"] + # this way we also have "None" if no resource_id is given yet: + resource_ids.append(str(resource_id)) + if resource_id is not None: + responses[resource_id] = job["actinia_core_response"] + job_status = { + "idpk_jobs": job["idpk_jobs"], + "actinia_core_jobid": str(resource_id), + "status": str(job["status"]) + } + jobs_status.append(job_status) + # status.append(str(job["status"])) + uuids.append(job["creation_uuid"]) + job_ids.append(str(job["idpk_jobs"])) + blocks.append(job["processing_block"]) + + # determine an overall batch status + overall_status_list = [job["status"] for job in jobs_status] + if "ERROR" in overall_status_list: + batch_status = "ERROR" + elif "TERMINATED" in overall_status_list: + if (("RUNNING" in overall_status_list) + or ("PENDING" in overall_status_list)): + batch_status = "TERMINATING" + else: + batch_status = "TERMINATED" + elif all(status == "SUCCESS" for status in overall_status_list): + batch_status = "SUCCESS" + elif all(status == "PREPARING" for status in overall_status_list): + batch_status = "PREPARING" + elif all((status == "PENDING" or status == "PREPARING") for status + in overall_status_list): + batch_status = "PENDING" + else: + batch_status = "RUNNING" + + # create block-wise statistics + processing_blocks = [] + for block in sorted(set(blocks)): + status_list = [job["status"] for job in jobs if + job["processing_block"] == block] + status_dict = _count_status_from_list(status_list) + if len(status_list) > 1: + parallel = len(status_list) + else: + parallel = 1 + + block_info = { + "block_num": block, + "parallel": parallel + } + block_stats = {**block_info, **status_dict} + processing_blocks.append(block_stats) + + # create summary statistics + summary_dict = { + "total": len(job_ids), + "status": _count_status_from_list(overall_status_list), + "blocks": processing_blocks + } + + # create overall response dict + responseDict = { + "actinia_gdi_batchid": batch_id, + "actinia_core_jobid": resource_ids, + # some parameters will be the same in all jobs: + "actinia_core_platform": jobs[0]["actinia_core_platform"], + "actinia_core_platform_name": jobs[0]["actinia_core_platform_name"], + "actinia_core_url": jobs[0]["actinia_core_url"], + "summary": summary_dict, + "actinia_core_response": responses, + "creation_uuids": uuids, + "idpk_jobs": job_ids, + "batch_description": jobs[0]["batch_description"], + "process": process, + "jobs_status": jobs_status, + "status": batch_status + } + return responseDict + + +def getAllBatchIds(): + """ Function to return all unique batch_ids from the database + """ + batch_ids_raw = set(getAllIds(batch=True)) + batch_ids = sorted([bid for bid in batch_ids_raw if bid is not None]) + return batch_ids + + +# def getAllBatches(process): +# """ Function to return all jobs that are part of a batch from the database +# """ +# result_list = [] +# batchids = getAllBatchIds() +# for batchid in batchids: +# jobs = getJobsByBatchId(batchid, process) +# jobs_response = createBatchResponseDict(jobs) +# result_list.append(jobs_response) +# result_dict = {"batch_jobs": result_list} +# return result_dict + + +def getJobsByBatchId(batch_id, process=None): + """ Function to return all jobs (db entries) via a batch_id and process + """ + filter_dict = {"batch_id": batch_id} + jobs = getAllJobs(filter_dict, process) + return jobs + + +# def startProcessingBlock(jobs, block): +# """ Function to start a specific processing block for an input list of +# jobs (db entries) +# """ +# jobs_to_start = [job for job in jobs if job["processing_block"] == block] +# jobs_responses = [] +# for job in jobs_to_start: +# start_kwargs = { +# "process": job["process"], +# "regeldatei": SingleJob(**job["job_description"]), +# "jobid": job["idpk_jobs"], +# "actinia_core_platform": job["actinia_core_platform"], +# "actinia_core_url": job["actinia_core_url"] +# } +# job_entry = startJobByActiniaType(**start_kwargs) +# jobs_responses.append(job_entry) +# return jobs_responses + + +def _count_status_from_list(input_list): + """ Function to count the occurence of different status strings + from a list + """ + lower_list = [item.lower() for item in input_list] + res_dict = { + "preparing": lower_list.count("preparing"), + "accepted": lower_list.count("pending"), + "running": lower_list.count("running"), + "finished": lower_list.count("success"), + "error": lower_list.count("error"), + "terminated": lower_list.count("terminated") + } + return res_dict diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py new file mode 100644 index 0000000..5e14c15 --- /dev/null +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Module to start the process Standortsicherung +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +from actinia_core.core.common.redis_interface import enqueue_job +from actinia_core.rest.base.resource_base import ResourceBase + + +# from actinia_gdi.api.common import checkConnectionWithoutResponse +# from actinia_gdi.core.actiniaCore import postActiniaCore, cancelActiniaCore +# from actinia_gdi.core.actiniaCore import parseActiniaIdFromUrl +# from actinia_gdi.core.gnosWriter import update +from actinia_parallel_plugin.core.jobtable import ( + # getJobById, + getJobByResource, + insertNewJob, + updateJobByID, +) +# from actinia_gdi.core.jobtable import insertNewJob, getJobById +# from actinia_gdi.core.jobtable import getAllIds, getAllJobs, cancelJobById +# from actinia_gdi.core.jobtable import getJobByResource +# from actinia_gdi.core.jobtable import updateJobWithActiniaByID +# from actinia_gdi.core.jobtable import updateJobWithTerraformByID +# from actinia_gdi.core.regeldatei import parseRulefile +# from actinia_gdi.model.regeldatei import Regeldatei +# from actinia_gdi.model.batchProcessChain import BatchProcessChain +# from actinia_gdi.resources.config import ACTINIACORE +# from actinia_gdi.resources.config import ACTINIACORE_VM +# from actinia_gdi.resources.config import PROCESSING +from actinia_parallel_plugin.resources.logging import log +# from actinia_gdi.core.terraformer import createVM, destroyVM +# from actinia_gdi.core.actiniaCore import shortenActiniaCoreResp + + +# def startJob(process, regeldatei, actinia, jobid): +# """ Starting job in running actinia-core instance and update job db """ +# job, err = getJobById(jobid) +# # url = job['actinia_core_url'] +# # platform = job['actinia_core_platform'] +# # connection = checkConnectionWithoutResponse(actinia, url) +# # if connection is not None: +# # actiniaCoreResp = postActiniaCore( +# # process, +# # regeldatei, +# # url, +# # platform +# # ) +# # log.debug(actiniaCoreResp) +# # status = actiniaCoreResp['status'] +# # +# # if status == 'error': +# # log.error("Error start processing in actinia-core") +# # +# # resourceId = parseActiniaIdFromUrl(actiniaCoreResp['resource_id']) +# # # initial actinia update, therefore with resourceId +# # job = updateJobWithActiniaByID( +# # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId +# # ) +# # return job +# # else: +# # job = updateJobWithActiniaByID( +# # jobid, 'error', None, +# # message=f"Error connection check to actinia ({url}) failed") +# # return job +# +# # TODO start job +# import pdb; pdb.set_trace() +# # initial actinia update, therefore with resourceId +# # job = updateJobWithActiniaByID( +# # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId +# # ) +# # return job + + +def insertJob(jsonDict, process, process_chain): + """ function to prepare and call InsertNewJob from regeldatei""" + # actinia_core_url = None + # actinia_core_platform = None + # actinia_core_platform_name = None + + try: + process_chain_struct = process_chain.to_struct() + except Exception as e: + log.error('Regeldatei is invalid!') + log.error(e) + return None + + # vm_procs = PROCESSING.actinia_vm_processes.replace( + # ' ', '').split(',') + # + # # set default actinia connection parameter + # if (process in vm_procs): + # actinia_core_url = None + # actinia_core_platform = 'vm' + # else: + # actinia_core_url = ACTINIACORE.url + # actinia_core_platform = 'openshift' + # + # # overwrite actinia connection parameter if set in rulefile + # if (regeldatei.processing_platform): + # if (regeldatei.processing_platform.lower() == 'vm'): + # actinia_core_url = None + # actinia_core_platform = 'vm' + # elif (regeldatei.processing_platform.lower() == 'openshift'): + # actinia_core_url = ACTINIACORE.url + # actinia_core_platform = 'openshift' + # + # # overwrite actinia connection parameter if set in rulefile + # if (regeldatei.processing_host): + # actinia_core_url = regeldatei.processing_host + # if not actinia_core_url.startswith('http'): + # actinia_core_url = ACTINIACORE_VM.scheme + '://' + actinia_core_url + # if len(actinia_core_url.split(':')) == 2: + # actinia_core_url += ':' + ACTINIACORE_VM.port + # + # if (regeldatei.processing_platform_name): + # actinia_core_platform_name = regeldatei.processing_platform_name + + job = insertNewJob( + jsonDict, + process_chain_struct, + process, + process_chain.feature_type, + # actinia_core_url, + # actinia_core_platform, + # actinia_core_platform_name + ) + return job + + +# def startJobByActiniaType(process, regeldatei, jobid, +# actinia_core_platform=None, actinia_core_url=None): +# """ Helper function to start job or create VM """ +# # if actinia_core_platform == 'vm' and actinia_core_url is None: +# # job = createVM(process, regeldatei, jobid) +# # elif actinia_core_platform == 'vm': +# # job = startJob(process, regeldatei, 'actinia-core-vm', jobid) +# # else: +# # job = startJob(process, regeldatei, 'actinia-core', jobid) +# job = startJob(process, regeldatei, 'actinia-core', jobid) +# return job + + +# def createJob(jsonDict, process): +# """ Method to parse regeldatei including fetching information from +# geonetwork and writing information to Jobtable +# as well as starting job in actinia-core +# +# This method can be called by HTTP POST +# @app.route('/processes/standortsicherung/jobs') +# """ +# +# if process == "netdefinition": +# regeldatei = BatchProcessChain(**jsonDict) +# regeldatei.feature_type = "null" +# else: +# regeldatei = parseRulefile(jsonDict) +# +# job = insertJob(jsonDict, process, regeldatei) +# jobid = job['idpk_jobs'] +# actinia_core_platform = job["actinia_core_platform"] +# actinia_core_url = job["actinia_core_url"] +# job = startJobByActiniaType(process=process, regeldatei=regeldatei, +# jobid=jobid, +# actinia_core_platform=actinia_core_platform, +# actinia_core_url=actinia_core_url) +# +# return job +# +# +# def getJob(jobid): +# """ Method to read job from Jobtable by id +# +# This method can be called by HTTP GET +# @app.route('/processes/standortsicherung/jobs/') +# """ +# +# job, err = getJobById(jobid) +# +# return job, err +# +# +# def getAllJobIDs(): +# """ Method to read all job ids from Jobtable +# +# This method can be called by HTTP GET +# @app.route('/processes/standortsicherung/jobs.html') +# """ +# +# job = getAllIds(batch=False) +# +# return job +# +# +# def getJobs(filters, process): +# """ Method to read all jobs from Jobtable with filter +# +# This method can be called by HTTP GET +# @app.route('/processes/standortsicherung/jobs') +# """ +# +# jobs = getAllJobs(filters, process) +# +# return jobs + +# status + +def shortenActiniaCoreResp(fullResp): + # replace webhook authentication with '***' + if 'process_chain_list' in fullResp: + if len(fullResp['process_chain_list']) > 0: + if 'webhooks' in fullResp['process_chain_list'][0]: + if 'auth' in fullResp['process_chain_list'][0]['webhooks']: + fullResp['process_chain_list'][0]['webhooks']['auth'] = \ + '***:***' + return fullResp + + +def updateJob(resource_id, actinia_resp, jobid): + """ Method to update job in Jobtable + + This method is called by webhook endpoint + """ + + status = actinia_resp["status"] + # record = getJobByResource("actinia_core_jobid", resource_id) + + # follow-up actinia update, therefore without resourceId + record = updateJobByID( + jobid, + status, + shortenActiniaCoreResp(actinia_resp) + ) + + # # TODO: for now if multiple records need to be updated (eg. for PT), this + # # can be told by specifying multiple uuids comma-separated in the + # # "feature_uuid" field of the rulefile. This might change later... + # if status == 'finished': + # try: + # gnosUuid = record['job_description']['feature_uuid'] + # utcnow = record['time_ended'] + # except Exception: + # log.warning('Feature has no uuid or time_ended') + # gnosUuid = None + # utcnow = None + # try: + # uuids = gnosUuid.split(',') + # for uuid in uuids: + # update(uuid, utcnow) + # except Exception: + # log.warning('Could not update geonetwork record') + + # # shutdown VM if process was calculated on VM + # terminate_status = ['finished', 'error', 'terminated'] + # processing_platform = record['actinia_core_platform'] + # process = record['process'] + # + # if (status in terminate_status + # and processing_platform is not None + # and processing_platform.lower() == 'vm' + # and 'processing_host' not in record['rule_configuration']): + # + # record = destroyVM(process, jobid) + + return record + + +# def updateJobByTerraform(terraformer_id, resp): +# """ Method to update job in Jobtable +# +# This method is called by webhook endpoint +# """ +# record = getJobByResource('terraformer_id', terraformer_id) +# jobid = record['idpk_jobs'] +# instance_ips = resp['instance_ips'] +# status = resp['status'] +# +# log.debug('Status of %s is %s' % (jobid, status)) +# if (status != 'STARTING' +# and (instance_ips is None or len(instance_ips) == 0)): +# log.error('Terraformer did not tell an IP for %s. Cannot start' +# ' processing or terminate', str(jobid)) +# record = updateJobWithTerraformByID( +# jobid, +# terraformer_id, +# resp, +# status='ERROR' +# ) +# return record +# +# if status != 'STARTING': +# if len(instance_ips) > 1: +# log.warning('Found more than one IP, only using last one') +# log.debug(instance_ips) +# # Currently we only start one VM for one dedicated job. Therefore we +# # assume, that only one VM was created and assign one IP to the +# # variable. If multiple VMs were created, we take the last IP +# for key, val in instance_ips.items(): +# actinia_core_ip = val +# +# actinia_core_url = (ACTINIACORE_VM.scheme + '://' + actinia_core_ip +# + ':' + ACTINIACORE_VM.port) +# else: +# actinia_core_url = None +# +# if status in ['PENDING', 'STARTING', 'STARTED', 'INSTALLING', 'RUNNING']: +# status = 'PREPARING' +# elif status == 'ERROR': +# status = 'ERROR' +# # shutdown VM +# process = record['process'] +# record = destroyVM(process, jobid) +# elif status == 'TERMINATING': +# status = None # keep actinia status +# elif status == 'TERMINATED': +# status = None # keep actinia status +# +# record = updateJobWithTerraformByID( +# jobid, +# terraformer_id, +# resp, +# actinia_core_url=actinia_core_url, +# status=status +# ) +# +# # start job in actinia +# if resp['status'] == 'RUNNING': +# process = record['process'] +# regeldatei = record['job_description'] +# rulefile = Regeldatei(**regeldatei) +# record = startJob(process, rulefile, 'actinia-core-vm', jobid) +# else: +# log.debug('Terraformer status is not "RUNNING", waiting further') +# +# return record +# +# +# def cancelJob(jobid): +# """ Method to cancel job from Jobtable by id +# +# This method can be called by HTTP POST +# @app.route('/processes/standortsicherung/jobs//operations/cancel') +# """ +# +# actiniacore = 'actinia-core' +# vm_needs_to_be_destroyed = False +# +# job, err = getJobById(jobid) +# +# if job is None: +# log.error(f"Error by requesting job with jobid {jobid}: {err['msg']}") +# return None +# +# status = job['status'] +# resourceId = job['actinia_core_jobid'] +# +# if not status or not resourceId: +# # the VM should be destroyed if the status is PREPARING and no resourcId +# # is set, e.g. if the starting of the VM fails without an ERROR +# if status == 'PREPARING' and not resourceId: +# pass +# else: +# log.error('Job status or resourceId is not set!') +# return None +# +# if resourceId: +# log.debug('Job status is ' + status + ' and resourceId is: ' + resourceId) +# url = job['actinia_core_url'] +# platform = job['actinia_core_platform'] +# +# if platform.lower() == 'vm': +# actiniacore = 'actinia-core-vm' +# vm_needs_to_be_destroyed = True +# +# if ('processing_host' in job['rule_configuration'] +# and job['rule_configuration']['processing_host'] is not None): +# vm_needs_to_be_destroyed = False +# +# if resourceId: +# connection = checkConnectionWithoutResponse(actiniacore, url) +# else: +# connection = None +# +# if status in ['PENDING', 'RUNNING'] and connection is not None: +# log.debug('Status is in PENDING or RUNNING, will cancel') +# res = cancelActiniaCore(resourceId, url, platform) +# if res: +# log.debug('Actinia-Core response TRUE') +# job = cancelJobById(jobid) +# log.debug('Job in jobtable is ' + job['status']) +# return job +# else: +# log.debug('Actinia-Core response is None') +# return None +# +# elif platform.lower() == 'vm' and vm_needs_to_be_destroyed is True: +# if connection is not None: +# log.debug('actinia-core connection exists and VM will ' + +# 'be destroyed') +# else: +# log.debug('actinia-core connection does not exist, but ' + +# 'VM will be destroyed') +# # destroy actinia-core VM +# record = destroyVM(job['process'], jobid) +# if record: +# log.debug('DestroyVM response exists') +# job = cancelJobById(jobid) +# log.debug('Job in jobtable is ' + job['status']) +# return job +# +# elif connection is not None: +# log.debug('Status not in PENDING or RUNNING and no VM to destroy, pass') +# return job +# +# else: +# log.error('There is no connection to actinia-core') +# return None diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 4ee1c73..52fd6d6 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -59,3 +59,358 @@ def applyMigrations(): with backend.lock(): backend.apply_migrations(backend.to_apply(migrations)) log.debug('Applied migrations.') + + +def getAllIds(batch=False): + """ Method to read all jobs from jobtabelle + + Args: + batch (bool): indicate whether only batch jobs should be read + + Returns: + jobIds (list): the record matching the id + """ + if batch is True: + field = JOBTABLE.batch_id_field + else: + field = JOBTABLE.id_field + with jobdb: + queryResult = Job.select(getattr(Job, field)).dicts() + + jobIds = [] + + # iterating reopens db connection!! + for i in queryResult: + jobIds.append(i[field]) + + # log.debug("Information read from jobtable.") + + jobdb.close() + + return jobIds + + +def getAllJobs(filters, process=None): + """ Method to read all jobs from jobtabelle with filter + + Args: filters (ImmutableMultiDict): the args from the HTTP call + + Returns: + jobs (list): the records matching the filter + """ + log.debug('Received query for jobs') + + if process == 'test': + query = Expression('a', '=', 'a') + elif process is None: + query = None + else: + query = Expression(getattr(Job, 'process'), '=', process) + + if filters: + log.debug("Found filters: " + str(filters)) + keys = [key for key in filters] + + for key in keys: + + try: + getattr(Job, key) + except Exception as e: + log.warning(str(e)) + continue + + log.debug("Filter " + str(key) + + " with value " + str(filters[key])) + + if isinstance(getattr(Job, key), AutoField): + try: + int(filters[key]) + except Exception as e: + log.error(str(e)) + jobdb.close() + return + + try: + # even though operators are listed as == and & in peewee docs, + # for Expression creation use '=' and 'AND'. + exp = Expression(getattr(Job, key), '=', filters[key]) + if query is not None: + query = Expression(query, 'AND', exp) + else: + query = exp + except AttributeError as e: + log.error(str(e)) + + with jobdb: + queryResult = Job.select().where(query).dicts() + + jobs = [] + # iterating reopens db connection!! + for i in queryResult: + jobs.append(i) + + log.info("Found " + str(len(jobs)) + " results for query.") + + jobdb.close() + + return jobs + + +def getJobById(jobid): + """ Method to read job from jobtabelle by id + + Args: + jobid (int): id of job + + Returns: + record (dict): the record matching the id + """ + try: + with jobdb: + queryResult = Job.select().where( + getattr(Job, JOBTABLE.id_field) == jobid).get() + record = model_to_dict(queryResult) + # log.info("Information read from jobtable for job with id " + # + str(record['idpk_jobs']) + ".") + err = None + except Job.DoesNotExist: + record = None + err = { + "status": 503, + "msg": "Either jobid does not exist or there was a " + "connection problem to the database. Please " + "try again later." + } + except OperationalError: + record = None + err = { + "status": 412, + "msg": "Database connection terminated abnormally before or " + "while processing the request. Please " + "try again later." + } + except Exception: + record = None + err = { + "status": 503, + "msg": "Either jobid does not exist or there was a " + "connection problem to the database. Please " + "try again later." + } + + jobdb.close() + + return record, err + + +def getJobByResource(key, val): + """ Method to read job from jobtabelle by resource + + Args: + key (string): key of attribute + val (string): value of attribute + + Returns: + record (dict): the record matching the id + """ + try: + with jobdb: + queryResult = Job.select().where( + getattr(Job, key) == val).get() + record = model_to_dict(queryResult) + # log.info("Information read from jobtable for job with id " + # + str(record['idpk_jobs']) + ".") + + except Job.DoesNotExist: + record = None + + jobdb.close() + + return record + + +def insertNewJob( + rule_configuration, + job_description, + process, + feature_type, + actinia_core_url=None, + actinia_core_platform=None, + actinia_core_platform_name=None + ): + """Insert new job into jobtabelle. + + Args: + rule_configuration (dict): original regeldatei + job_description (TODO): enriched regeldatei with geometadata + feature_type (string): feature_type name + actinia_core_url (string): url where processing will run + actinia_core_platform (string): platform where processing will run + + Returns: + record (dict): the new record + + """ + utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + creation_uuid = uuid4() + + job_kwargs = { + 'rule_configuration': rule_configuration, + 'job_description': job_description, + 'process': process, + 'status': 'PREPARING', + 'time_created': utcnow, + 'feature_type': feature_type, + 'actinia_core_url': actinia_core_url, + 'actinia_core_platform': actinia_core_platform, + 'actinia_core_platform_name': actinia_core_platform_name, + 'creation_uuid': creation_uuid + } + if "batch_id" in rule_configuration.keys(): + # then it's a batch job + job_kwargs["processing_block"] = rule_configuration["processing_block"] + job_kwargs["batch_id"] = rule_configuration["batch_id"] + job_kwargs["batch_description"] = rule_configuration[ + "batch_description"] + job = Job(**job_kwargs) + + with jobdb: + job.save() + # try to avoid "peewee.InterfaceError: connection already closed" + # so make each connection duration as short as possible + with jobdb: + queryResult = Job.select().where((Job.time_created == utcnow) & ( + Job.creation_uuid == creation_uuid)).get() + + record = model_to_dict(queryResult) + + log.info("Created new job with id " + str(record['idpk_jobs']) + ".") + + jobdb.close() + + return record + + +def updateJobByID( + jobid, status, resp, resourceId=None, message=None): + """ Method to update job in jobtabelle when processing status changed + + Args: + jobid (int): the id of the job + status (string): actinia-core processing status + resp (dict): actinia-core response + resourceId (str): actinia-core resourceId + message (str): general message for the job + + Returns: + updatedRecord (TODO): the updated record + """ + + # terraformer ["PENDING", "STARTING", "STARTED", "INSTALLING", "RUNNING", + # "ERROR", "TERMINATING", "TERMINATED"] + # terraformer ERROR leads to ERROR, else PREPARING or SUCCESS + # actinia-gdi ["PREPARING"] + # actinia-gdi ["PENDING", "RUNNING", "SUCCESS", "ERROR", "TERMINATED"] + # actinia-core [accepted, running, finished, error, terminated] + + if status == 'accepted': + status = 'PENDING' + elif status == 'running': + status = 'RUNNING' + elif status == 'finished': + status = 'SUCCESS' + elif status == 'error': + status = 'ERROR' + elif status == 'terminated': + status = 'TERMINATED' + + utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + record, err = getJobById(jobid) + dbStatus = record['status'] + + try: + # outcommented to see if more feasable if whole logs are passed + # if current_app.debug is False: + # smallRes = dict() + # smallRes['message'] = resp.get('message', None) + # smallRes['process_results'] = resp.get('process_results', None) + # resp = smallRes + + if status == 'PENDING': + if dbStatus == status: + return record + log.debug("Update status to " + status + " for job with id " + + str(record['idpk_jobs']) + ".") + updatekwargs = { + 'status': status, + 'actinia_core_response': resp, + 'actinia_core_jobid': resourceId + } + if message is not None: + updatekwargs['message'] = message + + query = Job.update(**updatekwargs).where( + getattr(Job, JOBTABLE.id_field) == jobid + ) + + elif status == 'RUNNING': + updatekwargs = dict() + + if dbStatus == status: + updatekwargs['actinia_core_response'] = resp + if message is not None: + updatekwargs['message'] = message + + else: + log.debug("Update status to " + status + " for job with id " + + str(record['idpk_jobs']) + ".") + updatekwargs['status'] = status + updatekwargs['actinia_core_response'] = resp + updatekwargs['time_started'] = utcnow + if message is not None: + updatekwargs['message'] = message + # TODO: check if time_estimated can be set + # time_estimated= + + query = Job.update(**updatekwargs).where( + getattr(Job, JOBTABLE.id_field) == jobid + ) + + elif status in ['SUCCESS', 'ERROR', 'TERMINATED']: + log.debug("Update status to " + status + " for job with id " + + str(record['idpk_jobs']) + ".") + updatekwargs = { + 'status': status, + 'actinia_core_response': resp, + 'time_ended': utcnow + } + if message is not None: + updatekwargs['message'] = message + + query = Job.update(**updatekwargs).where( + getattr(Job, JOBTABLE.id_field) == jobid + ) + + else: + log.error('Could not set the status to actinia-core status: ' + + status + '(Status not found.)') + return None + + with jobdb: + query.execute() + queryResult = Job.select().where( + getattr(Job, JOBTABLE.id_field) == jobid).get() + + record = model_to_dict(queryResult) + except Exception as e: + log.error('Could not set the status to actinia-core status: ' + status) + log.error(str(e)) + return None + + # log.debug("Updated status to " + status + " for job with id " + # + str(record['idpk_jobs']) + ".") + + jobdb.close() + + return record diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py new file mode 100644 index 0000000..527f984 --- /dev/null +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel processing +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +import sys +import traceback +import pickle +import json + +from actinia_core.processing.actinia_processing.ephemeral.persistent_processing import PersistentProcessing +from actinia_core.core.common.exceptions \ + import AsyncProcessError, AsyncProcessTermination, RsyncError +from actinia_core.core.common.exceptions import AsyncProcessTimeLimit +from actinia_core.models.response_models \ + import ProcessingResponseModel, ExceptionTracebackModel + +from actinia_parallel_plugin.core.batches import ( + checkProcessingBlockFinished, + getJobsByBatchId, +) +from actinia_parallel_plugin.core.jobs import updateJob + + +class ParallelPersistentProcessing(PersistentProcessing): + + def __init__(self, rdc, batch_id, processing_block, jobid): + super(ParallelPersistentProcessing, self).__init__(rdc) + self.batch_id = batch_id + self.processing_block = processing_block + self.jobid = jobid + + def _execute(self, process_chain, skip_permission_check=False): + """Overwrite this function in subclasses. + + This function will be executed by the run() function + + - Setup logger and credentials + - Analyse the process chain + - Create the temporal database + - Initialize the GRASS environment and create the temporary mapset + - Run the modules + - Parse the stdout output of the modules and generate the module results + + Args: + skip_permission_check (bool): If set True, the permission checks of + module access and process num + limits are not performed + + Raises: + This method will raise an AsyncProcessError, AsyncProcessTimeLimit + or AsyncProcessTermination + + """ + # Create the process chain + if self.rdc.iteration is not None: + process_list = \ + self._create_temporary_grass_environment_and_process_list_for_iteration( + process_chain=process_chain, + skip_permission_check=skip_permission_check) + else: + process_list = self._create_temporary_grass_environment_and_process_list( + process_chain=process_chain, + skip_permission_check=skip_permission_check) + + # Run all executables + self._execute_process_list(process_list=process_list) + # Parse the module sdtout outputs and create the results + self._parse_module_outputs() + + def run(self, process_chain): + """This function will run the processing and will catch and process + any Exceptions that were raised while processing. Call this function + to run the processing. + + You have to implement/overwrite two methods that are called here: + + * self._execute() + * self._final_cleanup() + + e_type, e_value, e_traceback = sys.exc_info() + message = [e.__class__, e_type, e_value, traceback.format_tb( + e_traceback)] + message = pprint.pformat(message) + """ + try: + # Run the _execute function that does all the work + self._execute(process_chain=process_chain) + except AsyncProcessTermination as e: + self.run_state = {"terminated": str(e)} + except AsyncProcessTimeLimit as e: + self.run_state = {"time limit exceeded": str(e)} + except AsyncProcessError as e: + e_type, e_value, e_tb = sys.exc_info() + model = ExceptionTracebackModel( + message=str(e_value), + traceback=traceback.format_tb(e_tb), + type=str(e_type) + ) + self.run_state = {"error": str(e), "exception": model} + except KeyboardInterrupt as e: + e_type, e_value, e_tb = sys.exc_info() + model = ExceptionTracebackModel( + message=str(e_value), + traceback=traceback.format_tb(e_tb), + type=str(e_type) + ) + self.run_state = {"error": str(e), "exception": model} + except Exception as e: + e_type, e_value, e_tb = sys.exc_info() + model = ExceptionTracebackModel( + message=str(e_value), + traceback=traceback.format_tb(e_tb), + type=str(e_type) + ) + self.run_state = {"error": str(e), "exception": model} + finally: + try: + # Call the final cleanup, before sending the status messages + self._final_cleanup() + except Exception as e: + e_type, e_value, e_tb = sys.exc_info() + model = ExceptionTracebackModel( + message=str(e_value), + traceback=traceback.format_tb(e_tb), + type=str(e_type) + ) + self.run_state = {"error": str(e), "exception": model} + # After all processing finished, send the final status + if "success" in self.run_state: + self._send_resource_finished(message=self.finish_message, + results=self.module_results) + elif "terminated" in self.run_state: + # Send an error message if an exception was raised + self._send_resource_terminated( + message=self.run_state["terminated"]) + elif "time limit exceeded" in self.run_state: + self._send_resource_time_limit_exceeded( + message=self.run_state["time limit exceeded"]) + elif "error" in self.run_state: + # Send an error message if an exception was raised + self._send_resource_error( + message=self.run_state["error"], + exception=self.run_state["exception"]) + else: + self._send_resource_error(message="Unknown error") + self._update_and_check_batch_jobs() + + def _update_and_check_batch_jobs(self): + """Checks batch jobs and starts new batch block if the current block + is successfully finished. + """ + + # update job to finished + resource_id = self.resource_id + response_data = self.resource_logger.get( + self.user_id, self.resource_id) + http_code, response_model = pickle.loads(response_data) + updateJob(resource_id, response_model, self.jobid) + + if "finished" == response_model["status"]: + jobs_from_batch = getJobsByBatchId( + self.batch_id, + "persistent" + ) + all_blocks = [ + job["processing_block"] for job in jobs_from_batch] + block = int(self.processing_block) + block_done = checkProcessingBlockFinished( + jobs_from_batch, block) + if block_done is True and block < max(all_blocks): + next_block = block + 1 + import pdb; pdb.set_trace() + print("TODO start next Block") + # next_jobs = startProcessingBlock(jobs_from_batch, + # next_block) + # res = createBatchResponseDict(next_jobs) + # return make_response(jsonify(res), 200) + + elif (response_model["status"] == "error" or + response_model["status"] == "terminated"): + # In this case, nothing happens and the next block is not + # started. + pass + + +def start_job(*args): + process_chain = json.loads(args[-1]) + processing = ParallelPersistentProcessing(*args[:-1]) + processing.run(process_chain) diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index 17e4685..8e6bcf9 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -25,8 +25,9 @@ __maintainer__ = "mundialis GmbH % Co. KG" +from actinia_parallel_plugin.api.batch import BatchJobsId from actinia_parallel_plugin.api.parallel_processing import \ - ParallelProcessingResource + AsyncParallelPersistentResource from actinia_parallel_plugin.core.jobtable import initJobDB, applyMigrations @@ -35,7 +36,22 @@ def create_endpoints(flask_api): apidoc = flask_api - apidoc.add_resource(ParallelProcessingResource, "/processing_parallel") + apidoc.add_resource( + AsyncParallelPersistentResource, + "/locations//mapsets/" + "/processing_parallel") + # apidoc.add_resource(ParallelProcessingResource, "/processing_parallel") + + # GET batch jobs by ID + apidoc.add_resource( + BatchJobsId, + "/processing_parallel/batchjobs/") + # "/processing_parallel/jobs/" + + # "/processing_parallel/batchjobs" + # "/processing_parallel/jobs" + + # initilalize jobtable initJobDB() diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py new file mode 100644 index 0000000..e5fcb5a --- /dev/null +++ b/src/actinia_parallel_plugin/model/batch.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2021-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Model class for batch +""" + +__license__ = "GPLv3" +__author__ = "Guido Riembauer, Anika Weinmann" +__copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import os +import json +from flask_restful_swagger_2 import Schema + +from actinia_core.models.process_chain import ProcessChainModel + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +rel_path = ("../apidocs/examples/" + + "batchjob_post_response_example.json") +print(script_dir) +print(rel_path) +abs_file_path = os.path.join(script_dir, rel_path) +print(abs_file_path) +print(os.path.isdir(abs_file_path)) +with open(abs_file_path) as jsonfile: + batchjob_post_docs_response_example = json.load(jsonfile) + + +class BatchJobsSummaryModel(Schema): + """Schema for a batchjob response summary""" + type = 'object' + properties = { + 'blocks': { + 'type': 'array', + 'description': 'Status summary for each processing blocks', + 'items': { + 'block_num': { + 'type': 'integer', + 'description': 'Number of processing block' + }, + 'parallel': { + 'type': 'integer', + 'description': ('Number of parallel jobs within ' + 'processing block') + }, + 'accepted': { + 'type': 'integer', + 'description': ('Number of jobs with actinia-core status ' + '"ACCEPTED" within block') + }, + 'error': { + 'type': 'integer', + 'description': ('Number of jobs with actinia-core status ' + '"ERROR" within block') + }, + 'finished': { + 'type': 'integer', + 'description': ('Number of jobs with actinia-core status ' + '"FINISHED" within block') + }, + 'preparing': { + 'type': 'integer', + 'description': ('Number of jobs with status ' + '"PREPARING" within block (jobs that ' + 'exist in the jobtable but have not yet ' + 'been posted to actinia-core)') + }, + 'running': { + 'type': 'integer', + 'description': ('Number of jobs with actinia-core status ' + '"RUNNING" within block') + }, + 'terminated': { + 'type': 'integer', + 'description': ('Number of jobs with actinia-core status ' + '"TERMINATED" within block') + }, + } + }, + 'status': { + 'type': 'array', + 'description': 'Status summary for all jobs', + 'items': { + 'accepted': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'actinia-core status "ACCEPTED"') + }, + 'error': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'actinia-core status "ERROR"') + }, + 'finished': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'actinia-core status "FINISHED"') + }, + 'preparing': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'status "PREPARING" (jobs that ' + 'exist in the jobtable but have not yet ' + 'been posted to actinia-core)') + }, + 'running': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'actinia-core status "RUNNING"') + }, + 'terminated': { + 'type': 'integer', + 'description': ('Overall number of jobs with ' + 'actinia-core status "TERMINATED"') + }, + } + }, + 'total': { + 'type': 'integer', + 'description': 'Overall number of jobs within batchjob' + } + } + + +class BatchProcessChainModel(Schema): + """Definition of the actinia-gdi batch process chain that includes several + actinia process chains that can be run in parallel or sequentially + """ + type = 'object' + properties = { + 'processing_host': { + 'type': 'string', + 'description': 'The actinia-core IP or URL in case the platform ' + 'is not OpenShift and no new VM should be created ' + 'by actinia-gdi' + }, + 'processing_platform': { + 'type': 'string', + 'description': 'The actinia-core platform, either "openshift" or ' + '"vm". If platform is "vm" and no actinia_core_url ' + 'is given, actinia-gdi will create a new VM.' + }, + 'processing_platform_name': { + 'type': 'string', + 'description': 'The actinia-core platform name. Only used to ' + 'match a job to a VM if VM not started by ' + 'actinia-gdi. Ideally it would contain the job ' + 'type (actinia-core-pt or actinia-core-oc) and ' + 'a unique ID.' + }, + 'jobs': {'type': 'array', + 'items': ProcessChainModel, + 'description': "A list of process chains (jobs) that should " + "be executed in parallel or sequentially " + "in the order provided by the list."} + } + + +class BatchJobResponseModel(Schema): + """Response schema for creating and requesting the status of a Batchjob + """ + type = 'object' + properties = { + 'actinia_core_jobid': { + 'type': 'array', + 'description': ('The actinia-core resource IDs for all individual ' + 'jobs'), + 'items': {'type': 'string'} + }, + 'actinia_core_platform': { + 'type': 'string', + 'description': ('The actinia-core platform, either "openshift"' + ' or "vm"') + }, + 'actinia_core_platform_name': { + 'type': 'string', + 'description': 'The actinia-core platform name' + }, + 'actinia_core_response': { + 'type': 'array', + 'description': 'The responses of actinia-core for individual jobs', + 'items': {'type': 'object'} + }, + 'actinia_core_url': { + 'type': 'string', + 'description': ('The actinia-core IP or URL where actinia-core ' + 'is processing the batchjob') + }, + 'actinia_gdi_batchid': { + 'type': 'integer', + 'description': 'The batch ID' + }, + 'batch_description': BatchProcessChainModel, + 'creation_uuid': { + 'type': 'array', + 'description': ('Unique ids for the individual jobs at creation ' + 'time before idpk_jobs is known. ' + '(More unique than creation timestamp)'), + 'items': {'type': 'string'} + }, + 'idpk_jobs': { + 'type': 'array', + 'description': 'The individual job IDs', + 'items': {'type': 'integer'} + }, + 'process': { + 'type': 'string', + 'description': 'The process of the job, e.g. netdefinition' + }, + 'status': { + 'type': 'string', + 'description': 'The overall status of the batchjob', + 'enum': [ + "PREPARING", + "PENDING", + "RUNNING", + "SUCCESS", + "ERROR", + "TERMINATING", + "TERMINATED"] + }, + 'jobs_status': { + 'type': 'array', + 'description': ('Status of the individual Jobs by actinia-core ' + 'resource ID and job ID'), + 'items': { + 'actinia_core_job_id': { + 'type': 'string', + 'description': 'The actinia-core resource ID for the job' + }, + 'idpk_jobs': { + 'type': 'integer', + 'description': 'The job ID' + }, + 'status': { + 'type': 'string', + 'description': 'Status of the Job', + 'enum': [ + "PREPARING", + "PENDING", + "RUNNING", + "SUCCESS", + "ERROR", + "TERMINATED" + ] + } + } + }, + + 'summary': BatchJobsSummaryModel + } + example = batchjob_post_docs_response_example diff --git a/src/actinia_parallel_plugin/model/batch_process_chain.py b/src/actinia_parallel_plugin/model/batch_process_chain.py new file mode 100644 index 0000000..f481a49 --- /dev/null +++ b/src/actinia_parallel_plugin/model/batch_process_chain.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2021-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Model classes for Batch Process Chain +""" + +__license__ = "GPLv3" +__author__ = "Julia Haas, Guido Riembauer" +__copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +from jsonmodels import models, fields + + +class ModuleStdOut(models.Base): + """Model for object in BatchProcessChain + + Model for optional stdout in module + """ + id = fields.StringField(required=True) + format = fields.StringField(required=True) + delimiter = fields.StringField(required=True) + + +class ModuleExport(models.Base): + """Model for object in BatchProcessChain + + Model for optional export in output + """ + format = fields.StringField() + type = fields.StringField() + + +class ModuleOutput(models.Base): + """Model for object in BatchProcessChain + + Model for each output in module outputs array + """ + param = fields.StringField() + value = fields.StringField() + export = fields.EmbeddedField(ModuleExport) + + +class ModuleInput(models.Base): + """Model for object in BatchProcessChain + + Model for each input in module inputs array + """ + param = fields.StringField(required=True) + value = fields.StringField(required=True) + + +class Module(models.Base): + """Model for object in BatchProcessChain + + Model for each module in module list array + """ + module = fields.StringField(required=True) # string + id = fields.StringField(required=True) # string + inputs = fields.ListField([ModuleInput]) # array of objects + flags = fields.StringField() # string + stdout = fields.EmbeddedField(ModuleStdOut) # string + outputs = fields.ListField([ModuleOutput]) + + +class Job(models.Base): + """Model for object in BatchProcessChain + + Model for each job in jobs array + """ + version = fields.StringField() # string + parallel = fields.StringField(required=True) # bool + list = fields.ListField([Module], required=True) # array of objects + # the block and batch id is not in the json but is filled later + processing_block = fields.IntField() + batch_id = fields.IntField() + + +class BatchProcessChain(models.Base): + """Model for BatchProcessChain + Including all information for all jobs and general information on + processing platform and host + This is used by the sequential processing netdefinition/jobs endpoint + """ + + processing_platform = fields.StringField() # string + processing_platform_name = fields.StringField() # string + processing_host = fields.StringField() # string + jobs = fields.ListField([Job], required=True) # array of objects + + +class SingleJob(models.Base): + """Model for SingleJob + Including all information for all modules and general information on + processing platform and host + This is used by the parallel processing netdefinition/batchjobs endpoint + and is only created internally for individual jobs of a BatchProcessChain + in order to have the processing_platform etc. attributes attached to each + job as well. + """ + version = fields.StringField() # string + list = fields.ListField([Module], required=True) # array of objects + # the following are given by the user to the BatchProcessChain and are + # then internally applied to each SingleJob + processing_platform = fields.StringField() # string + processing_platform_name = fields.StringField() # string + processing_host = fields.StringField() # string + # processing_block, batch_description and + # batch id are not in the json but is filled later + processing_block = fields.IntField() + batch_id = fields.IntField() + # batch_description holds the entire batch processing chain + batch_description = fields.EmbeddedField(BatchProcessChain) diff --git a/test_postbodies/parallel_processing.json b/test_postbodies/parallel_processing.json new file mode 100644 index 0000000..925f2ba --- /dev/null +++ b/test_postbodies/parallel_processing.json @@ -0,0 +1,63 @@ +{ + "jobs": [ + { + "list": [ + { + "module": "r.mapcalc", + "id": "r_mapcalc_0_nonparallel", + "inputs":[ + {"param": "expression", "value": "baum = elevation@PERMANENT * 2"} + ] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_1_parallel", + "inputs":[ + {"param": "n", "value": "228500"}, + {"param": "s", "value": "215000"}, + {"param": "e", "value": "645000"}, + {"param": "w", "value": "637500"} + ] + }, + { + "module": "r.mapcalc", + "id": "r_mapcalc_1_parallel", + "inputs":[ + {"param": "expression", "value": "baum2 = baum@test_parallel * 2"} + ] + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_2_parallel", + "inputs":[ + {"param": "n", "value": "228500"}, + {"param": "s", "value": "215000"}, + {"param": "e", "value": "637500"}, + {"param": "w", "value": "630000"} + ] + }, + { + "module": "r.mapcalc", + "id": "r_mapcalc_2_parallel", + "inputs":[ + {"param": "expression", "value": "baum2 = baum@test_parallel * 2"} + ] + } + ], + "parallel": "true", + "version": "1" + } + ] +} diff --git a/test_postbodies/parallel_processing_soll.json b/test_postbodies/parallel_processing_soll.json new file mode 100644 index 0000000..9722de4 --- /dev/null +++ b/test_postbodies/parallel_processing_soll.json @@ -0,0 +1,138 @@ +{ + "processing_host": "http://actinia-core-docker:8088/", + "processing_platform_name": "example_name", + "jobs": [ + { + "list": [ + { + "module": "stac_importer", + "inputs":[ + {"param": "param1", "value": "value1"} + ] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "module": "actinia_tiling", + "comment": "All jobs executed in parallel loop for each tile", + "inputs":[ + // With this approach, also area size would be possible + // {"param": "size", "value": "10000"}, + {"param": "num_tiles", "value": "10000"} + ], + "outputs":[ + {"param": "raster", "value": "ndvi,ndwi"} + ], + "jobs": [ + { + "list": [ + { + "module": "g.region", + "inputs":[ + {"param": "x", "value": "{{ tile_id }}"} + ] + }, + { + "module": "r.mask", + "inputs":[ + {"param": "x", "value": "y"} + ] + } + ], + "parallel": "false" + }, + { + "list": [ + { + "module": "r.mapcalc", + "inputs":[ + {"param": "x", "value": "ndvi"} + ] + } + ], + "parallel": "true" + }, + { + "list": [ + { + "module": "r.mapcalc", + "inputs":[ + {"param": "x", "value": "ndwi"} + ] + } + ], + "parallel": "true" + } + ] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "module": "actinia_tiling", + "comment": "All jobs executed in parallel loop for each tile", + "inputs":[ + // With this approach, also area size would be possible + // {"param": "size", "value": "10000"}, + {"param": "num_tiles", "value": "10000"}, + // TODO: parameter or flag? + {"param": "reuse_tiles", "value": "true"} + ], + "outputs":[ + {"param": "raster", "value": "agg1,agg2"} + ], + "jobs": [ + { + "list": [ + { + "module": "g.region", + "inputs":[ + {"param": "x", "value": "{{ tile_id }}"} + ] + }, + { + "module": "r.mask", + "inputs":[ + {"param": "x", "value": "y"} + ] + } + ], + "parallel": "false" + }, + { + "list": [ + { + "module": "t.aggregate", + "inputs":[ + {"param": "x", "value": "red_nir,green_red"} + ] + } + ], + "parallel": "true" + }, + { + "list": [ + { + "module": "t.aggregate", + "inputs":[ + {"param": "x", "value": "blue,blue_nir"} + ] + } + ], + "parallel": "true" + } + ] + } + ], + "parallel": "false", + "version": "1" + } + ] +} From af276c11c3d588745ffca1a5cd5b071e7756980b Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 19 May 2022 06:34:37 +0200 Subject: [PATCH 03/47] zwischenstand 2: resourceID is set --- src/actinia_parallel_plugin/api/batch.py | 4 +- src/actinia_parallel_plugin/api/job.py | 78 ++++ .../api/parallel_processing.py | 33 +- src/actinia_parallel_plugin/apidocs/jobs.py | 317 +++++++++++++++ .../apidocs/regeldatei.py | 363 ++++++++++++++++++ src/actinia_parallel_plugin/core/jobs.py | 33 +- src/actinia_parallel_plugin/core/jobtable.py | 4 + .../core/parallel_processing_job.py | 137 +++++++ .../core/persistent_processing.py | 13 +- src/actinia_parallel_plugin/endpoints.py | 7 + src/actinia_parallel_plugin/model/batch.py | 21 +- .../model/response_models.py | 33 ++ 12 files changed, 999 insertions(+), 44 deletions(-) create mode 100644 src/actinia_parallel_plugin/api/job.py create mode 100644 src/actinia_parallel_plugin/apidocs/jobs.py create mode 100644 src/actinia_parallel_plugin/apidocs/regeldatei.py create mode 100644 src/actinia_parallel_plugin/core/parallel_processing_job.py diff --git a/src/actinia_parallel_plugin/api/batch.py b/src/actinia_parallel_plugin/api/batch.py index 5ffc306..612b405 100644 --- a/src/actinia_parallel_plugin/api/batch.py +++ b/src/actinia_parallel_plugin/api/batch.py @@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -Module to communicate with jobtable +Endpoint definitions for batch job """ __license__ = "GPLv3" @@ -55,7 +55,7 @@ def get(self, batchid): log.info(("\n Received HTTP GET request for batch" f" with id {str(batchid)}")) - jobs = getJobsByBatchId(batch) + jobs = getJobsByBatchId(batchid) if len(jobs) == 0: res = (jsonify(SimpleResponseModel( status=404, diff --git a/src/actinia_parallel_plugin/api/job.py b/src/actinia_parallel_plugin/api/job.py new file mode 100644 index 0000000..e94a229 --- /dev/null +++ b/src/actinia_parallel_plugin/api/job.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Endpoint definitions for job +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +from flask_restful import Resource +from flask_restful_swagger_2 import swagger +from flask import make_response, jsonify + +from actinia_core.models.response_models import \ + SimpleResponseModel + +from actinia_parallel_plugin.resources.logging import log +from actinia_parallel_plugin.core.jobs import ( + getJob, +) +from actinia_parallel_plugin.apidocs import jobs + + +class JobId(Resource): + """ Definition for endpoint standortsicherung + @app.route('/processes/standortsicherung/jobs/') + + Contains HTTP GET endpoint reading a job + Contains swagger documentation + """ + + @swagger.doc(jobs.jobId_get_docs) + def get(self, jobid): + """ Wrapper method to receive HTTP call and pass it to function + + This method is called by HTTP GET + @app.route('/processes/standortsicherung/jobs/') + This method is calling core method readJob + """ + if jobid is None: + return make_response("No jobid was given", 400) + + log.info("\n Received HTTP GET request for job with id " + str(jobid)) + + job, err = getJob(jobid) + + if job is not None: + return make_response(jsonify(job), 200) + else: + res = (jsonify(SimpleResponseModel( + status=err["status"], + message=err["msg"] + ))) + return make_response(res, err["status"]) + + def post(self, jobid): + res = jsonify(SimpleResponseModel( + status=405, + message="Method Not Allowed" + )) + return make_response(res, 405) diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index 6d8dfeb..386dca4 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -49,14 +49,14 @@ from actinia_parallel_plugin.model.response_models import ( SimpleStatusCodeResponseModel, ) -from actinia_parallel_plugin.model.batch_process_chain import ( - # BatchProcessChain, - SingleJob, -) -from actinia_parallel_plugin.core.jobtable import updateJobByID -from actinia_parallel_plugin.core.jobs import updateJob +# from actinia_parallel_plugin.model.batch_process_chain import ( +# SingleJob, +# ) +# from actinia_parallel_plugin.core.jobtable import updateJobByID +# from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log from actinia_parallel_plugin.core.persistent_processing import start_job +from actinia_parallel_plugin.core.parallel_processing_job import AsyncParallelJobResource class AsyncParallelPersistentResource(ResourceBase): @@ -71,7 +71,7 @@ def __init__(self): @swagger.doc(helloworld.describeHelloWorld_get_docs) def get(self, location_name, mapset_name): """Get 'Hello world!' as answer string.""" - return SimpleStatusCodeResponseModel(status=200, message=self.msg) + return SimpleStatusCodeResponseModel(status=200, message="TEST") # def prepare_actinia(self): # e.g. start a VM and check connection to actinia-core on it @@ -83,8 +83,9 @@ def _start_job(self, process, process_chain, jobid): # TODO prepare_actinia ? # TODO execute_actinia ? # TODO goodby_actinia ? -# has_json = False -# self.request_data = pc + + # has_json = False + # self.request_data = pc rdc = self.preprocess( has_json=True, @@ -134,6 +135,7 @@ def _start_processing_block(self, jobs, block): process_chain = dict() process_chain["list"] = job["rule_configuration"]["list"] process_chain["version"] = job["rule_configuration"]["version"] + jobid = job["idpk_jobs"] start_kwargs = { "process": job["process"], # "pc": SingleJob(**job["job_description"]), @@ -142,7 +144,16 @@ def _start_processing_block(self, jobs, block): # "actinia_core_platform": job["actinia_core_platform"], # "actinia_core_url": job["actinia_core_url"] } - job_entry = self._start_job(**start_kwargs) + parallel_job = AsyncParallelJobResource( + post_url=self.post_url, + process_chain=process_chain, + location_name=self.location_name, + mapset_name=self.mapset_name, + batch_id=self.batch_id, + job_id=jobid + ) + parallel_job.start_job("persistent", 1) + job_entry = parallel_job.get_job_entry() jobs_responses.append(job_entry) return jobs_responses @@ -154,6 +165,7 @@ def post(self, location_name, mapset_name): self.location_name = location_name self.mapset_name = mapset_name + # import pdb; pdb.set_trace() json_dict = request.get_json(force=True) log.info("Received HTTP POST with batchjob: %s" % @@ -173,7 +185,6 @@ def post(self, location_name, mapset_name): # start first processing block first_jobs = self._start_processing_block(jobs_in_db, 1) - import pdb; pdb.set_trace() first_status = [entry["status"] for entry in first_jobs] all_jobs = getJobsByBatchId(self.batch_id, "persistent") if None in first_jobs: diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py new file mode 100644 index 0000000..79180ec --- /dev/null +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Documentation objects for generic job endpoints +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +from flask_restful_swagger_2 import Schema + +from actinia_parallel_plugin.apidocs.regeldatei import ( + RegeldateiModel, + ProcessesProcOutputModel, +) + + +class EnrichedProcInputBaseModel(Schema): + type = 'object' + properties = { + 'name': { + 'type': 'string', + 'description': 'Name of input data' + }, + 'type': { + 'type': 'string', + 'enum': ["GNOS", "DATABASE", "PARAMETER", "STATUS"], + 'description': 'Type of input. Can be "GNOS", "DATABASE", ' + + '"PARAMETER" or "STATUS"' + }, + 'geodata_meta': "string" # TODO: GeodataResponseModel + } + + +class EnrichedProcInputGnosModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + EnrichedProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + # }, + # ProcessesProcInputGnosModel, + # { + # '$ref': '#/definitions/ProcessesProcInputGnosModel' + } + + ] + + +class EnrichedProcInputDatabaseModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + EnrichedProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + # }, + # ProcessesProcInputDatabaseModel, + # { + # '$ref': '#/definitions/ProcessesProcInputDatabaseModel' + } + ] + + +class EnrichedProcInputParameterModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + EnrichedProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + # }, + # ProcessesProcInputParameterModel, + # { + # '$ref': '#/definitions/ProcessesProcInputParameterModel' + } + ] + + +class EnrichedProcInputStatusModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + EnrichedProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + # }, + # ProcessesProcInputStatusModel, + # { + # '$ref': '#/definitions/ProcessesProcInputStatusModel' + } + ] + + +class EnrichedProcInputModel(Schema): + """Request schema for creating a job""" + type = 'object' + # TODO: use oneOf (was not parsed in petstore) + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + EnrichedProcInputGnosModel, + EnrichedProcInputDatabaseModel, + EnrichedProcInputParameterModel, + EnrichedProcInputStatusModel, + { + '$ref': '#/definitions/EnrichedProcInputGnosModel' + }, + { + '$ref': '#/definitions/EnrichedProcInputDatabaseModel' + }, + { + '$ref': '#/definitions/EnrichedProcInputParameterModel' + }, + { + '$ref': '#/definitions/EnrichedProcInputStatusModel' + } + ] + + +class EnrichedProcModel(Schema): + """Request schema for creating a job""" + type = 'object' + properties = { + 'name': { + 'type': 'integer', + 'description': 'Name of process' + }, + 'input': { + 'type': 'array', + 'description': 'Definitions for process input data', + 'items': EnrichedProcInputModel + }, + 'output': { + 'type': 'array', + 'description': 'Definitions for process output data', + 'items': ProcessesProcOutputModel + }, + 'dependsOn': { + 'type': 'string', + 'description': 'List of names of processes on which this process' + + ' depends on. See also "status" as input parameter' + } + } + + +class EnrichedRegeldateiModel(Schema): + """Request schema for creating a job""" + type = 'object' + properties = { + 'rule_area_id': { + 'type': 'integer', + 'description': 'Identifier of area where Regeldatei is valid' + }, + 'rule_area': { + 'type': 'string', + 'description': 'Name of area where Regeldatei is valid' + }, + 'feature_type': { + 'type': 'string', + 'description': 'Name of feature type to run job with' + }, + 'feature_uuid': { + 'type': 'string', + 'description': 'Geonetwork UUID of feature type to run job with' + }, + 'feature_source': EnrichedProcInputBaseModel, + 'processing_platform': { + 'type': 'string', + 'description': 'The actinia-core platform, either "openshift" or "vm". If platform is "vm" and no actinia_core_url is given, actinia-gdi will create a new VM.' + }, + 'processing_platform_name': { + 'type': 'string', + 'description': 'The actinia-core platform name. Only used to match a job to a VM if VM not started by actinia-gdi. Ideally it would contain the job type (actinia-core-pt or actinia-core-oc) and a unique ID.' + }, + 'processing_host': { + 'type': 'string', + 'description': 'The actinia-core IP or URL in case the platform is not OpenShift and no new VM should be created by actinia-gdi' + }, + 'procs': { + 'type': 'array', + 'description': 'List of processes to run', + 'items': EnrichedProcModel + }, + 'geodata_meta': "string" # TODO: GeodataResponseModel + } + # TODO add example + # example = jobs_post_docs_request_example + required = ["feature_source", "procs"] + + +class ProcessesJobResponseModel(Schema): + """Response schema for creating a job""" + type = 'object' + properties = { + 'idpk_jobs': { + 'type': 'integer', + 'description': 'The job ID' + }, + 'process': { + 'type': 'string', + 'description': 'The process of the job, e.g standortsicherung ' + + 'or potentialtrenches' + }, + 'feature_type': { + 'type': 'string', + 'description': 'The feature type of the job' + }, + 'rule_configuration': RegeldateiModel, + 'job_description': EnrichedRegeldateiModel, + 'time_created': { + 'type': 'string', + 'description': 'Timestamp when job was created' + }, + 'time_started': { + 'type': 'string', + 'description': 'Timestamp when job was created' + }, + 'time_estimated': { + 'type': 'string', + 'description': 'Timestamp when job was created' + }, + 'time_ended': { + 'type': 'string', + 'description': 'Timestamp when job was created' + }, + 'metadata': { + 'type': 'string', + 'description': 'Not specified yet' + }, + 'status': { + 'type': 'string', + 'description': 'Status of the Job', + 'enum': [ + "PENDING", + "RUNNING", + "SUCCESS", + "ERROR", + "TERMINATED" + ] + }, + 'actinia_core_response': { + 'type': 'object', + 'description': 'The Response of actinia-core at creation time' + }, + 'actinia_core_jobid': { + 'type': 'string', + 'description': 'The actinia-core resource ID for the job' + }, + 'actinia_core_platform': { + 'type': 'string', + 'description': 'The actinia-core platform, either "openshift" or "vm"' + }, + 'actinia_core_platform_name': { + 'type': 'string', + 'description': 'The actinia-core platform name' + }, + 'actinia_core_url': { + 'type': 'string', + 'description': 'The actinia-core IP or URL where actinia-core is processing the job' + }, + 'creation_uuid': { + 'type': 'string', + 'description': 'A unique id for the job at creation time before idpk_jobs is known. (More unique than creation timestamp)' + }, + 'terraformer_id': { + 'type': 'string', + 'description': 'The terraformer instances ID for the job' + }, + 'terraformer_response': { + 'type': 'string', + 'description': 'The Response/Status of terraformer' + } + } + # TODO add example + # example = jobs_post_docs_response_example + + +jobId_get_docs = { + "summary": "Returns job by jobid.", + "description": "This request will get the requested job from the jobtable", + "tags": [ + "processing" + ], + "parameters": [ + { + "in": "path", + "name": "jobid", + "type": "string", + "description": "a jobid", + "required": True + } + ], + "responses": { + "200": { + "description": "The job object of the requested id", + "schema": ProcessesJobResponseModel + } + } +} diff --git a/src/actinia_parallel_plugin/apidocs/regeldatei.py b/src/actinia_parallel_plugin/apidocs/regeldatei.py new file mode 100644 index 0000000..8c642de --- /dev/null +++ b/src/actinia_parallel_plugin/apidocs/regeldatei.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Model classes for regeldatei +""" + +__license__ = "GPLv3" +__author__ = "Carmen Tawalika, Anika Weinmann" +__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import os +import json + +from flask_restful_swagger_2 import Schema + +from actinia_parallel_plugin.model.response_models import GeodataResponseModel + + +# script_dir = os.path.dirname(os.path.abspath(__file__)) +# null = "null" +# +# rel_path = ("../apidocs/examples/" + +# "standortsicherung_jobs_post_request_example.json") +# abs_file_path = os.path.join(script_dir, rel_path) +# with open(abs_file_path) as jsonfile: +# sos_jobs_post_docs_request_example = json.load(jsonfile) +# +# rel_path = ("../apidocs/examples/" + +# "pt_jobs_post_request_example.json") +# abs_file_path = os.path.join(script_dir, rel_path) +# with open(abs_file_path) as jsonfile: +# pt_jobs_post_docs_request_example = json.load(jsonfile) + + +class ProcessesProcInputBaseModel(Schema): + type = 'object' + properties = { + 'name': { + 'type': 'string', + 'description': 'Name of input data' + }, + 'type': { + 'type': 'string', + 'enum': ["GNOS", "DATABASE", "PARAMETER", "STATUS"], + 'description': 'Type of input. Can be "GNOS", "DATABASE", ' + + '"PARAMETER" or "STATUS"' + } + } + + +class ProcessesFilterModel(Schema): + type = 'object' + properties = { + 'name': { + 'type': 'string', + 'description': 'Not yet implemented or specified' + }, + 'metadata_field': { + 'type': 'string', + 'description': 'Not yet implemented or specified' + }, + 'metadata_value': { + 'type': 'string', + 'description': 'Not yet implemented or specified' + }, + 'operator': { + 'type': 'string', + 'description': 'Not yet implemented or specified' + } + } + + +class ProcessesProcInputGnosModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + }, + { + 'type': 'object', + 'properties': { + 'tags': { + 'type': 'array', + 'description': 'Geonetwork tags to filter data', + 'items': { + 'type': 'string' + } + }, + 'uuid': { + 'type': 'string', + 'description': 'Geonetwork UUID to identify certain record' + }, + 'attributes': { + 'type': 'array', + 'description': 'Database attribute of data source to use', + 'items': { + 'type': 'string' + } + }, + 'filter': { + 'type': 'array', + 'description': 'Not yet implemented or specified', + 'items': ProcessesFilterModel + } + } + } + + ] + + +class ProcessesProcInputDatabaseModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + }, + { + 'type': 'object', + 'properties': { + 'table': { + 'type': 'string', + 'description': 'Database connection string with table' + }, + 'attributes': { + 'type': 'array', + 'description': 'Database attribute of data source to use', + 'items': { + 'type': 'string' + } + }, + 'filter': { + 'type': 'array', + 'description': 'Not yet implemented or specified', + 'items': ProcessesFilterModel + } + } + } + ] + + +class ProcessesProcInputParameterModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + }, + { + 'type': 'object', + 'properties': { + 'value': { + 'type': 'array', + 'description': 'Array of input parameter. Can be int, ' + + 'float or string', + 'items': { + # TODO: find out how to allow multiple types + # 'type': 'int, float, string' + } + }, + 'uom': { + 'type': 'string', + 'description': 'Unit of measurement for values' + } + } + } + ] + + +class ProcessesProcInputFileModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + }, + { + 'type': 'object', + 'properties': { + 'value': { + 'type': 'array', + 'description': 'Array of input parameter. Can be int, ' + + 'float or string', + 'items': { + # TODO: find out how to allow multiple types + # 'type': 'int, float, string' + } + } + } + } + ] + + +class ProcessesProcInputStatusModel(Schema): + """Request schema for creating a job""" + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputBaseModel, + { + '$ref': '#/definitions/ProcessesProcInputBaseModel' + }, + { + 'type': 'object', + 'properties': { + 'status': { + 'type': 'string', + 'description': 'Status of another process as input. See ' + + 'also "dependsOn"' + } + } + } + ] + + +class ProcessesProcInputModel(Schema): + """Definitions for process input data""" + type = 'object' + # TODO: use oneOf (was not parsed in petstore) + allOf = [ + # keep this line, otherwise BaseModel does not exist in document + ProcessesProcInputGnosModel, + ProcessesProcInputDatabaseModel, + ProcessesProcInputParameterModel, + ProcessesProcInputFileModel, + ProcessesProcInputStatusModel, + { + '$ref': '#/definitions/ProcessesProcInputGnosModel' + }, + { + '$ref': '#/definitions/ProcessesProcInputDatabaseModel' + }, + { + '$ref': '#/definitions/ProcessesProcInputParameterModel' + }, + { + '$ref': '#/definitions/ProcessesProcInputFileModel' + }, + { + '$ref': '#/definitions/ProcessesProcInputStatusModel' + } + ] + + +class ProcessesProcOutputModel(Schema): + """Definitions for process output data""" + type = 'object' + properties = { + 'name': { + 'type': 'string', + 'description': 'Name of attribute column in output datasource' + }, + 'type': { + 'type': 'string', + 'description': 'Column type of attribut column' + }, + 'default': { + 'type': 'string', + 'description': 'Default value of attribut column' + }, + 'value': { + 'type': 'string', + 'description': 'Filename if output is file' + } + } + + +class ProcessesProcModel(Schema): + """List of processes to run""" + type = 'object' + properties = { + 'name': { + 'type': 'integer', + 'description': 'Name of process' + }, + 'input': { + 'type': 'array', + 'description': 'Definitions for process input data', + 'items': ProcessesProcInputModel + }, + 'output': { + 'type': 'array', + 'description': 'Definitions for process output data', + 'items': ProcessesProcOutputModel + }, + 'dependsOn': { + 'type': 'string', + 'description': 'List of names of processes on which this process' + + ' depends on. See also "status" as input parameter' + } + } + + +class RegeldateiModel(Schema): + """Request schema to create a job""" + type = 'object' + properties = { + 'rule_area_id': { + 'type': 'integer', + 'description': 'Identifier of area where Regeldatei is valid' + }, + 'rule_area': { + 'type': 'string', + 'description': 'Name of area where Regeldatei is valid' + }, + 'feature_type': { + 'type': 'string', + 'description': 'Name of feature type to run job with' + }, + 'feature_uuid': { + 'type': 'string', + 'description': 'Geonetwork UUID of feature type to run job with' + }, + 'feature_source': ProcessesProcInputModel, + 'processing_platform': { + 'type': 'string', + 'description': 'The actinia-core platform, either "openshift" or "vm". If platform is "vm" and no actinia_core_url is given, actinia-gdi will create a new VM.' + }, + 'processing_platform_name': { + 'type': 'string', + 'description': 'The actinia-core platform name. Only used to match a job to a VM if VM not started by actinia-gdi. Ideally it would contain the job type (actinia-core-pt or actinia-core-oc) and a unique ID.' + }, + 'processing_host': { + 'type': 'string', + 'description': 'The actinia-core IP or URL in case the platform is not OpenShift and no new VM should be created by actinia-gdi' + }, + 'procs': { + 'type': 'array', + 'description': 'List of processes to run', + 'items': ProcessesProcModel + }, + 'geodata_meta': GeodataResponseModel + } + # examples = { + # 'zero': { + # 'value': sos_jobs_post_docs_request_example + # }, + # 'max': { + # 'value': pt_jobs_post_docs_request_example + # } + # } + # example = sos_jobs_post_docs_request_example + required = ["feature_source", "procs"] diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index 5e14c15..3f5de10 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -34,7 +34,7 @@ # from actinia_gdi.core.actiniaCore import parseActiniaIdFromUrl # from actinia_gdi.core.gnosWriter import update from actinia_parallel_plugin.core.jobtable import ( - # getJobById, + getJobById, getJobByResource, insertNewJob, updateJobByID, @@ -189,20 +189,20 @@ def insertJob(jsonDict, process, process_chain): # actinia_core_url=actinia_core_url) # # return job -# -# -# def getJob(jobid): -# """ Method to read job from Jobtable by id -# -# This method can be called by HTTP GET -# @app.route('/processes/standortsicherung/jobs/') -# """ -# -# job, err = getJobById(jobid) -# -# return job, err -# -# + + +def getJob(jobid): + """ Method to read job from Jobtable by id + + This method can be called by HTTP GET + @app.route('/processes/standortsicherung/jobs/') + """ + + job, err = getJobById(jobid) + + return job, err + + # def getAllJobIDs(): # """ Method to read all job ids from Jobtable # @@ -252,7 +252,8 @@ def updateJob(resource_id, actinia_resp, jobid): record = updateJobByID( jobid, status, - shortenActiniaCoreResp(actinia_resp) + shortenActiniaCoreResp(actinia_resp), + resourceId=resource_id ) # # TODO: for now if multiple records need to be updated (eg. for PT), this diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 52fd6d6..e7aeed5 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -370,6 +370,8 @@ def updateJobByID( updatekwargs['time_started'] = utcnow if message is not None: updatekwargs['message'] = message + if resourceId is not None: + updatekwargs['actinia_core_jobid'] = resourceId # TODO: check if time_estimated can be set # time_estimated= @@ -387,6 +389,8 @@ def updateJobByID( } if message is not None: updatekwargs['message'] = message + if resourceId is not None: + updatekwargs['actinia_core_jobid'] = resourceId query = Job.update(**updatekwargs).where( getattr(Job, JOBTABLE.id_field) == jobid diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py new file mode 100644 index 0000000..ea5cbec --- /dev/null +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-present mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel processing job +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import json +import pickle +from flask import request, make_response, jsonify +from flask_restful_swagger_2 import swagger +# from flask_restful_swagger_2 import Resource + +from actinia_core.models.response_models import \ + SimpleResponseModel +from actinia_core.rest.base.resource_base import ResourceBase +from actinia_core.core.common.redis_interface import enqueue_job + +from actinia_parallel_plugin.apidocs import helloworld +from actinia_parallel_plugin.core.batches import ( + createBatch, + createBatchId, + createBatchResponseDict, + getJobsByBatchId, + # startProcessingBlock, +) +from actinia_parallel_plugin.core.jobtable import ( + getJobById, +) +from actinia_parallel_plugin.model.response_models import ( + SimpleStatusCodeResponseModel, +) +# from actinia_parallel_plugin.model.batch_process_chain import ( +# SingleJob, +# ) +# from actinia_parallel_plugin.core.jobtable import updateJobByID +from actinia_parallel_plugin.core.jobs import updateJob +from actinia_parallel_plugin.resources.logging import log +from actinia_parallel_plugin.core.persistent_processing import start_job + + +class AsyncParallelJobResource(ResourceBase): + """Job for parallel processing""" + + def __init__(self, post_url, process_chain, location_name, mapset_name, + batch_id, job_id): + super(AsyncParallelJobResource, self).__init__(post_url) + self.location_name = location_name + self.mapset_name = mapset_name + self.batch_id = batch_id + self.job_id = job_id + self.request_data = process_chain + + def start_job(self, process, block): + """Starting job in running actinia-core instance and update job db.""" + job, err = getJobById(self.job_id) + # TODO prepare_actinia ? + # TODO execute_actinia ? + # TODO goodby_actinia ? + + # has_json = False + # self.request_data = pc + + rdc = self.preprocess( + has_json=False, + location_name=self.location_name, + mapset_name=self.mapset_name + ) + if rdc: + from actinia_parallel_plugin.core.persistent_processing import \ + ParallelPersistentProcessing + processing = ParallelPersistentProcessing( + rdc, self.batch_id, block, self.job_id) + # processing.run(self.request_data) + processing.run() + + # enqueue_job( + # self.job_timeout, + # start_job, + # rdc, + # self.batch_id, + # block, + # jobid, + # json.dumps(process_chain) + # ) + + # update job in jobtable to update the resource id + response_data = self.resource_logger.get( + self.user_id, self.resource_id) + http_code, response_model = pickle.loads(response_data) + job = updateJob(processing.resource_id, response_model, self.job_id) + return job + + def get_job_entry(self): + """Return job entry by requesting jobtable from db.""" + return getJobById(self.job_id)[0] + + # first_jobs = self._start_processing_block(jobs_in_db, 1) + def _start_processing_block(self, jobs, block): + """Starts first processing block of jobs from batch process. + """ + jobs_to_start = [ + job for job in jobs if job["processing_block"] == block] + jobs_responses = [] + for job in jobs_to_start: + process_chain = dict() + process_chain["list"] = job["rule_configuration"]["list"] + process_chain["version"] = job["rule_configuration"]["version"] + start_kwargs = { + "process": job["process"], + "process_chain": process_chain, + "jobid": job["idpk_jobs"], + # "actinia_core_platform": job["actinia_core_platform"], + # "actinia_core_url": job["actinia_core_url"] + } + job_entry = self._start_job(**start_kwargs) + jobs_responses.append(job_entry) + return jobs_responses diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py index 527f984..397c7fd 100644 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -52,7 +52,8 @@ def __init__(self, rdc, batch_id, processing_block, jobid): self.processing_block = processing_block self.jobid = jobid - def _execute(self, process_chain, skip_permission_check=False): + # def _execute(self, process_chain, skip_permission_check=False): + def _execute(self, skip_permission_check=False): """Overwrite this function in subclasses. This function will be executed by the run() function @@ -78,11 +79,11 @@ def _execute(self, process_chain, skip_permission_check=False): if self.rdc.iteration is not None: process_list = \ self._create_temporary_grass_environment_and_process_list_for_iteration( - process_chain=process_chain, + # process_chain=process_chain, skip_permission_check=skip_permission_check) else: process_list = self._create_temporary_grass_environment_and_process_list( - process_chain=process_chain, + # process_chain=process_chain, skip_permission_check=skip_permission_check) # Run all executables @@ -90,7 +91,8 @@ def _execute(self, process_chain, skip_permission_check=False): # Parse the module sdtout outputs and create the results self._parse_module_outputs() - def run(self, process_chain): + # def run(self, process_chain): + def run(self): """This function will run the processing and will catch and process any Exceptions that were raised while processing. Call this function to run the processing. @@ -107,7 +109,8 @@ def run(self, process_chain): """ try: # Run the _execute function that does all the work - self._execute(process_chain=process_chain) + # self._execute(process_chain=process_chain) + self._execute() except AsyncProcessTermination as e: self.run_state = {"terminated": str(e)} except AsyncProcessTimeLimit as e: diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index 8e6bcf9..bc95373 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -26,6 +26,7 @@ from actinia_parallel_plugin.api.batch import BatchJobsId +from actinia_parallel_plugin.api.job import JobId from actinia_parallel_plugin.api.parallel_processing import \ AsyncParallelPersistentResource from actinia_parallel_plugin.core.jobtable import initJobDB, applyMigrations @@ -46,6 +47,12 @@ def create_endpoints(flask_api): apidoc.add_resource( BatchJobsId, "/processing_parallel/batchjobs/") + + # GET job by ID + apidoc.add_resource( + JobId, + "/processing_parallel/jobs/") + # "/processing_parallel/jobs/" # "/processing_parallel/batchjobs" diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index e5fcb5a..3b626e6 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -32,15 +32,16 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -rel_path = ("../apidocs/examples/" - + "batchjob_post_response_example.json") -print(script_dir) -print(rel_path) -abs_file_path = os.path.join(script_dir, rel_path) -print(abs_file_path) -print(os.path.isdir(abs_file_path)) -with open(abs_file_path) as jsonfile: - batchjob_post_docs_response_example = json.load(jsonfile) +# TODO +# rel_path = ("../apidocs/examples/" +# + "batchjob_post_response_example.json") +# print(script_dir) +# print(rel_path) +# abs_file_path = os.path.join(script_dir, rel_path) +# print(abs_file_path) +# print(os.path.isdir(abs_file_path)) +# with open(abs_file_path) as jsonfile: +# batchjob_post_docs_response_example = json.load(jsonfile) class BatchJobsSummaryModel(Schema): @@ -266,4 +267,4 @@ class BatchJobResponseModel(Schema): 'summary': BatchJobsSummaryModel } - example = batchjob_post_docs_response_example + # example = batchjob_post_docs_response_example diff --git a/src/actinia_parallel_plugin/model/response_models.py b/src/actinia_parallel_plugin/model/response_models.py index 92c9621..511b139 100644 --- a/src/actinia_parallel_plugin/model/response_models.py +++ b/src/actinia_parallel_plugin/model/response_models.py @@ -49,3 +49,36 @@ class SimpleStatusCodeResponseModel(Schema): status=200, message="success" ) SimpleStatusCodeResponseModel.example = simpleResponseExample + + +class GeodataResponseModel(Schema): + """Model for object in enriched Regeldatei + + This object contains the metadata for input geodata from GNOS + """ + type = 'object' + properties = { + 'uuid': { + 'type': 'string', + 'description': 'The Geonetwork uuid.' + }, + 'bbox': { + 'type': 'array', + 'items': { + 'type': 'number' + }, + 'minItems': 4, + 'maxItems': 4, + 'description': 'The bounding box of the result.' + }, + 'crs': { + 'type': 'string', + 'description': 'The coordinate reference system of the result.' + }, + 'table': { + 'type': 'string', + 'description': ('The database connection string of the source ' + + 'of the result.') + } + } + required = ["uuid", "bbox"] From 656576213f501b001792928ee70c7a09c4e21fa9 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 20 May 2022 07:18:04 +0200 Subject: [PATCH 04/47] zwischenstand --- src/actinia_parallel_plugin/api/batch.py | 2 +- .../api/parallel_processing.py | 182 +++++------- src/actinia_parallel_plugin/apidocs/batch.py | 3 - .../apidocs/regeldatei.py | 4 +- src/actinia_parallel_plugin/core/batches.py | 69 +++-- src/actinia_parallel_plugin/core/example.py | 31 -- src/actinia_parallel_plugin/core/jobs.py | 281 +----------------- .../core/parallel_processing_job.py | 126 ++++---- .../core/persistent_processing.py | 33 +- src/actinia_parallel_plugin/endpoints.py | 4 - src/actinia_parallel_plugin/model/batch.py | 2 +- test_postbodies/parallel_processing.json | 44 ++- 12 files changed, 249 insertions(+), 532 deletions(-) delete mode 100644 src/actinia_parallel_plugin/core/example.py diff --git a/src/actinia_parallel_plugin/api/batch.py b/src/actinia_parallel_plugin/api/batch.py index 612b405..106f513 100644 --- a/src/actinia_parallel_plugin/api/batch.py +++ b/src/actinia_parallel_plugin/api/batch.py @@ -26,7 +26,7 @@ from flask_restful import Resource from flask_restful_swagger_2 import swagger -from flask import make_response, jsonify, request +from flask import make_response, jsonify from actinia_core.models.response_models import \ SimpleResponseModel diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index 386dca4..ea90274 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -41,11 +41,11 @@ createBatchId, createBatchResponseDict, getJobsByBatchId, - # startProcessingBlock, -) -from actinia_parallel_plugin.core.jobtable import ( - getJobById, + startProcessingBlock, ) +# from actinia_parallel_plugin.core.jobtable import ( +# getJobById, +# ) from actinia_parallel_plugin.model.response_models import ( SimpleStatusCodeResponseModel, ) @@ -55,8 +55,8 @@ # from actinia_parallel_plugin.core.jobtable import updateJobByID # from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log -from actinia_parallel_plugin.core.persistent_processing import start_job -from actinia_parallel_plugin.core.parallel_processing_job import AsyncParallelJobResource +# from actinia_parallel_plugin.core.persistent_processing import start_job +# from actinia_parallel_plugin.core.batches import startProcessingBlock class AsyncParallelPersistentResource(ResourceBase): @@ -77,85 +77,72 @@ def get(self, location_name, mapset_name): # e.g. start a VM and check connection to actinia-core on it # return things - def _start_job(self, process, process_chain, jobid): - """Starting job in running actinia-core instance and update job db.""" - job, err = getJobById(jobid) - # TODO prepare_actinia ? - # TODO execute_actinia ? - # TODO goodby_actinia ? - - # has_json = False - # self.request_data = pc - - rdc = self.preprocess( - has_json=True, - location_name=self.location_name, - mapset_name=self.mapset_name - ) - if rdc: - block = 1 - from actinia_parallel_plugin.core.persistent_processing import \ - ParallelPersistentProcessing - processing = ParallelPersistentProcessing( - rdc, self.batch_id, block, jobid) - processing.run(process_chain) - - # enqueue_job( - # self.job_timeout, - # start_job, - # rdc, - # self.batch_id, - # block, - # jobid, - # json.dumps(process_chain) - # ) - - # html_code, response_model = pickle.loads(self.response_data) - # job = updateJob(resourceId, actiniaCoreResp, jobid) - # job = updateJobByID( - # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId - # ) - job = getJobById(jobid)[0] - return job - # return make_response(jsonify(response_model), html_code) - - # initial actinia update, therefore with resourceId - # job = updateJobWithActiniaByID( - # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId - # ) - # return job - - def _start_processing_block(self, jobs, block): - """Starts first processing block of jobs from batch process. - """ - jobs_to_start = [ - job for job in jobs if job["processing_block"] == block] - jobs_responses = [] - for job in jobs_to_start: - process_chain = dict() - process_chain["list"] = job["rule_configuration"]["list"] - process_chain["version"] = job["rule_configuration"]["version"] - jobid = job["idpk_jobs"] - start_kwargs = { - "process": job["process"], - # "pc": SingleJob(**job["job_description"]), - "process_chain": process_chain, - "jobid": job["idpk_jobs"], - # "actinia_core_platform": job["actinia_core_platform"], - # "actinia_core_url": job["actinia_core_url"] - } - parallel_job = AsyncParallelJobResource( - post_url=self.post_url, - process_chain=process_chain, - location_name=self.location_name, - mapset_name=self.mapset_name, - batch_id=self.batch_id, - job_id=jobid - ) - parallel_job.start_job("persistent", 1) - job_entry = parallel_job.get_job_entry() - jobs_responses.append(job_entry) - return jobs_responses + # def _start_job(self, process, process_chain, jobid): + # """Starting job in running actinia-core instance and update job db.""" + # job, err = getJobById(jobid) + # # TODO prepare_actinia ? + # # TODO execute_actinia ? + # # TODO goodby_actinia ? + # + # # has_json = False + # # self.request_data = pc + # + # rdc = self.preprocess( + # has_json=True, + # location_name=self.location_name, + # mapset_name=self.mapset_name + # ) + # if rdc: + # block = 1 + # from actinia_parallel_plugin.core.persistent_processing import \ + # ParallelPersistentProcessing + # processing = ParallelPersistentProcessing( + # rdc, self.batch_id, block, jobid) + # processing.run(process_chain) + # + # # enqueue_job( + # # self.job_timeout, + # # start_job, + # # rdc, + # # self.batch_id, + # # block, + # # jobid, + # # json.dumps(process_chain) + # # ) + # + # job = getJobById(jobid)[0] + # return job + + # def _start_processing_block(self, jobs, block): + # """Starts first processing block of jobs from batch process. + # """ + # jobs_to_start = [ + # job for job in jobs if job["processing_block"] == block] + # jobs_responses = [] + # for job in jobs_to_start: + # process_chain = dict() + # process_chain["list"] = job["rule_configuration"]["list"] + # process_chain["version"] = job["rule_configuration"]["version"] + # jobid = job["idpk_jobs"] + # start_kwargs = { + # "process": job["process"], + # "process_chain": process_chain, + # "jobid": job["idpk_jobs"], + # # "actinia_core_platform": job["actinia_core_platform"], + # # "actinia_core_url": job["actinia_core_url"] + # } + # parallel_job = AsyncParallelJobResource( + # post_url=self.post_url, + # process_chain=process_chain, + # location_name=self.location_name, + # mapset_name=self.mapset_name, + # batch_id=self.batch_id, + # job_id=jobid + # ) + # parallel_job.start_job("persistent", 1) + # job_entry = parallel_job.get_job_entry() + # jobs_responses.append(job_entry) + # return jobs_responses # TODO get all batch jobs @swagger.doc(helloworld.describeHelloWorld_get_docs) @@ -165,7 +152,7 @@ def post(self, location_name, mapset_name): self.location_name = location_name self.mapset_name = mapset_name - # import pdb; pdb.set_trace() + self.post_url = self.api_info["request_url"] json_dict = request.get_json(force=True) log.info("Received HTTP POST with batchjob: %s" % @@ -184,7 +171,15 @@ def post(self, location_name, mapset_name): return make_response(res, 500) # start first processing block - first_jobs = self._start_processing_block(jobs_in_db, 1) + # first_jobs = self._start_processing_block(jobs_in_db, 1) + first_jobs = startProcessingBlock( + jobs_in_db, + 1, + self.batch_id, + self.location_name, + self.mapset_name, + self.post_url + ) first_status = [entry["status"] for entry in first_jobs] all_jobs = getJobsByBatchId(self.batch_id, "persistent") if None in first_jobs: @@ -200,16 +195,3 @@ def post(self, location_name, mapset_name): else: return make_response(jsonify(createBatchResponseDict(all_jobs)), 412) - - # # TODO start a parallel processing job as batch job - # @swagger.doc(helloworld.describeHelloWorld_post_docs) - # def post(self): - # """Hello World post method with name from postbody.""" - # - # req_data = request.get_json(force=True) - # if isinstance(req_data, dict) is False or "name" not in req_data: - # return make_response("Missing name in JSON content", 400) - # name = req_data["name"] - # msg = transform_input(name) - # - # return SimpleStatusCodeResponseModel(status=200, message=msg) diff --git a/src/actinia_parallel_plugin/apidocs/batch.py b/src/actinia_parallel_plugin/apidocs/batch.py index 2122f7d..4435190 100644 --- a/src/actinia_parallel_plugin/apidocs/batch.py +++ b/src/actinia_parallel_plugin/apidocs/batch.py @@ -28,9 +28,6 @@ from actinia_core.models.response_models import \ SimpleResponseModel -from actinia_parallel_plugin.model.response_models import ( - SimpleStatusCodeResponseModel, -) from actinia_parallel_plugin.model.batch import ( BatchJobResponseModel, ) diff --git a/src/actinia_parallel_plugin/apidocs/regeldatei.py b/src/actinia_parallel_plugin/apidocs/regeldatei.py index 8c642de..42c193e 100644 --- a/src/actinia_parallel_plugin/apidocs/regeldatei.py +++ b/src/actinia_parallel_plugin/apidocs/regeldatei.py @@ -24,8 +24,8 @@ __copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" -import os -import json +# import os +# import json from flask_restful_swagger_2 import Schema diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 8c6420e..0dd0f33 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -24,21 +24,17 @@ __copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" +from json import loads -from actinia_parallel_plugin.core.jobtable import getAllIds +from actinia_parallel_plugin.core.jobs import insertJob +from actinia_parallel_plugin.core.jobtable import getAllIds, getAllJobs +from actinia_parallel_plugin.core.parallel_processing_job import \ + AsyncParallelJobResource from actinia_parallel_plugin.model.batch_process_chain import ( BatchProcessChain, SingleJob, ) -# from actinia_gdi.core.jobtable import getAllIds, getAllJobs from actinia_parallel_plugin.resources.logging import log -from actinia_parallel_plugin.core.jobs import insertJob -from actinia_parallel_plugin.core.jobtable import getAllJobs -# from actinia_gdi.resources.config import JOBTABLE -# from actinia_parallel_plugin.core.jobs import startJobByActiniaType -# from actinia_gdi.core.jobs import startJobByActiniaType, cancelJob - -from json import loads def assignProcessingBlocks(jsonDict): @@ -291,23 +287,44 @@ def getJobsByBatchId(batch_id, process=None): return jobs -# def startProcessingBlock(jobs, block): -# """ Function to start a specific processing block for an input list of -# jobs (db entries) -# """ -# jobs_to_start = [job for job in jobs if job["processing_block"] == block] -# jobs_responses = [] -# for job in jobs_to_start: -# start_kwargs = { -# "process": job["process"], -# "regeldatei": SingleJob(**job["job_description"]), -# "jobid": job["idpk_jobs"], -# "actinia_core_platform": job["actinia_core_platform"], -# "actinia_core_url": job["actinia_core_url"] -# } -# job_entry = startJobByActiniaType(**start_kwargs) -# jobs_responses.append(job_entry) -# return jobs_responses +def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, + post_url): + """ Function to start a specific processing block for an input list of + jobs (db entries) + """ + jobs_to_start = [ + job for job in jobs if job["processing_block"] == block] + jobs_responses = [] + mapset_suffix = "" + if len(jobs_to_start) > 1: + mapset_suffix = "_parallel_" + for num, job in enumerate(jobs_to_start): + process_chain = dict() + process_chain["list"] = job["rule_configuration"]["list"] + process_chain["version"] = job["rule_configuration"]["version"] + jobid = job["idpk_jobs"] + start_kwargs = { + "process": job["process"], + "process_chain": process_chain, + "jobid": job["idpk_jobs"], + # "actinia_core_platform": job["actinia_core_platform"], + # "actinia_core_url": job["actinia_core_url"] + } + mapset_name_parallel = mapset_name + if mapset_suffix != "": + mapset_name_parallel += f"{mapset_suffix}{num}" + parallel_job = AsyncParallelJobResource( + post_url=post_url, + process_chain=process_chain, + location_name=location_name, + mapset_name=mapset_name_parallel, + batch_id=batch_id, + job_id=jobid + ) + parallel_job.start_parallel_job("persistent", 1) + job_entry = parallel_job.get_job_entry() + jobs_responses.append(job_entry) + return jobs_responses def _count_status_from_list(input_list): diff --git a/src/actinia_parallel_plugin/core/example.py b/src/actinia_parallel_plugin/core/example.py deleted file mode 100644 index 55d82e7..0000000 --- a/src/actinia_parallel_plugin/core/example.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-present mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Example core functionality -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - - -def transform_input(inp): - """Example core function""" - out = f"Hello world {inp.upper()}!" - return out diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index 3f5de10..eb7cc11 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -24,75 +24,13 @@ __copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" - -from actinia_core.core.common.redis_interface import enqueue_job -from actinia_core.rest.base.resource_base import ResourceBase - - -# from actinia_gdi.api.common import checkConnectionWithoutResponse -# from actinia_gdi.core.actiniaCore import postActiniaCore, cancelActiniaCore -# from actinia_gdi.core.actiniaCore import parseActiniaIdFromUrl -# from actinia_gdi.core.gnosWriter import update from actinia_parallel_plugin.core.jobtable import ( getJobById, - getJobByResource, + # getJobByResource, insertNewJob, updateJobByID, ) -# from actinia_gdi.core.jobtable import insertNewJob, getJobById -# from actinia_gdi.core.jobtable import getAllIds, getAllJobs, cancelJobById -# from actinia_gdi.core.jobtable import getJobByResource -# from actinia_gdi.core.jobtable import updateJobWithActiniaByID -# from actinia_gdi.core.jobtable import updateJobWithTerraformByID -# from actinia_gdi.core.regeldatei import parseRulefile -# from actinia_gdi.model.regeldatei import Regeldatei -# from actinia_gdi.model.batchProcessChain import BatchProcessChain -# from actinia_gdi.resources.config import ACTINIACORE -# from actinia_gdi.resources.config import ACTINIACORE_VM -# from actinia_gdi.resources.config import PROCESSING from actinia_parallel_plugin.resources.logging import log -# from actinia_gdi.core.terraformer import createVM, destroyVM -# from actinia_gdi.core.actiniaCore import shortenActiniaCoreResp - - -# def startJob(process, regeldatei, actinia, jobid): -# """ Starting job in running actinia-core instance and update job db """ -# job, err = getJobById(jobid) -# # url = job['actinia_core_url'] -# # platform = job['actinia_core_platform'] -# # connection = checkConnectionWithoutResponse(actinia, url) -# # if connection is not None: -# # actiniaCoreResp = postActiniaCore( -# # process, -# # regeldatei, -# # url, -# # platform -# # ) -# # log.debug(actiniaCoreResp) -# # status = actiniaCoreResp['status'] -# # -# # if status == 'error': -# # log.error("Error start processing in actinia-core") -# # -# # resourceId = parseActiniaIdFromUrl(actiniaCoreResp['resource_id']) -# # # initial actinia update, therefore with resourceId -# # job = updateJobWithActiniaByID( -# # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId -# # ) -# # return job -# # else: -# # job = updateJobWithActiniaByID( -# # jobid, 'error', None, -# # message=f"Error connection check to actinia ({url}) failed") -# # return job -# -# # TODO start job -# import pdb; pdb.set_trace() -# # initial actinia update, therefore with resourceId -# # job = updateJobWithActiniaByID( -# # jobid, status, shortenActiniaCoreResp(actiniaCoreResp), resourceId -# # ) -# # return job def insertJob(jsonDict, process, process_chain): @@ -151,46 +89,6 @@ def insertJob(jsonDict, process, process_chain): return job -# def startJobByActiniaType(process, regeldatei, jobid, -# actinia_core_platform=None, actinia_core_url=None): -# """ Helper function to start job or create VM """ -# # if actinia_core_platform == 'vm' and actinia_core_url is None: -# # job = createVM(process, regeldatei, jobid) -# # elif actinia_core_platform == 'vm': -# # job = startJob(process, regeldatei, 'actinia-core-vm', jobid) -# # else: -# # job = startJob(process, regeldatei, 'actinia-core', jobid) -# job = startJob(process, regeldatei, 'actinia-core', jobid) -# return job - - -# def createJob(jsonDict, process): -# """ Method to parse regeldatei including fetching information from -# geonetwork and writing information to Jobtable -# as well as starting job in actinia-core -# -# This method can be called by HTTP POST -# @app.route('/processes/standortsicherung/jobs') -# """ -# -# if process == "netdefinition": -# regeldatei = BatchProcessChain(**jsonDict) -# regeldatei.feature_type = "null" -# else: -# regeldatei = parseRulefile(jsonDict) -# -# job = insertJob(jsonDict, process, regeldatei) -# jobid = job['idpk_jobs'] -# actinia_core_platform = job["actinia_core_platform"] -# actinia_core_url = job["actinia_core_url"] -# job = startJobByActiniaType(process=process, regeldatei=regeldatei, -# jobid=jobid, -# actinia_core_platform=actinia_core_platform, -# actinia_core_url=actinia_core_url) -# -# return job - - def getJob(jobid): """ Method to read job from Jobtable by id @@ -203,31 +101,6 @@ def getJob(jobid): return job, err -# def getAllJobIDs(): -# """ Method to read all job ids from Jobtable -# -# This method can be called by HTTP GET -# @app.route('/processes/standortsicherung/jobs.html') -# """ -# -# job = getAllIds(batch=False) -# -# return job -# -# -# def getJobs(filters, process): -# """ Method to read all jobs from Jobtable with filter -# -# This method can be called by HTTP GET -# @app.route('/processes/standortsicherung/jobs') -# """ -# -# jobs = getAllJobs(filters, process) -# -# return jobs - -# status - def shortenActiniaCoreResp(fullResp): # replace webhook authentication with '***' if 'process_chain_list' in fullResp: @@ -287,155 +160,3 @@ def updateJob(resource_id, actinia_resp, jobid): # record = destroyVM(process, jobid) return record - - -# def updateJobByTerraform(terraformer_id, resp): -# """ Method to update job in Jobtable -# -# This method is called by webhook endpoint -# """ -# record = getJobByResource('terraformer_id', terraformer_id) -# jobid = record['idpk_jobs'] -# instance_ips = resp['instance_ips'] -# status = resp['status'] -# -# log.debug('Status of %s is %s' % (jobid, status)) -# if (status != 'STARTING' -# and (instance_ips is None or len(instance_ips) == 0)): -# log.error('Terraformer did not tell an IP for %s. Cannot start' -# ' processing or terminate', str(jobid)) -# record = updateJobWithTerraformByID( -# jobid, -# terraformer_id, -# resp, -# status='ERROR' -# ) -# return record -# -# if status != 'STARTING': -# if len(instance_ips) > 1: -# log.warning('Found more than one IP, only using last one') -# log.debug(instance_ips) -# # Currently we only start one VM for one dedicated job. Therefore we -# # assume, that only one VM was created and assign one IP to the -# # variable. If multiple VMs were created, we take the last IP -# for key, val in instance_ips.items(): -# actinia_core_ip = val -# -# actinia_core_url = (ACTINIACORE_VM.scheme + '://' + actinia_core_ip -# + ':' + ACTINIACORE_VM.port) -# else: -# actinia_core_url = None -# -# if status in ['PENDING', 'STARTING', 'STARTED', 'INSTALLING', 'RUNNING']: -# status = 'PREPARING' -# elif status == 'ERROR': -# status = 'ERROR' -# # shutdown VM -# process = record['process'] -# record = destroyVM(process, jobid) -# elif status == 'TERMINATING': -# status = None # keep actinia status -# elif status == 'TERMINATED': -# status = None # keep actinia status -# -# record = updateJobWithTerraformByID( -# jobid, -# terraformer_id, -# resp, -# actinia_core_url=actinia_core_url, -# status=status -# ) -# -# # start job in actinia -# if resp['status'] == 'RUNNING': -# process = record['process'] -# regeldatei = record['job_description'] -# rulefile = Regeldatei(**regeldatei) -# record = startJob(process, rulefile, 'actinia-core-vm', jobid) -# else: -# log.debug('Terraformer status is not "RUNNING", waiting further') -# -# return record -# -# -# def cancelJob(jobid): -# """ Method to cancel job from Jobtable by id -# -# This method can be called by HTTP POST -# @app.route('/processes/standortsicherung/jobs//operations/cancel') -# """ -# -# actiniacore = 'actinia-core' -# vm_needs_to_be_destroyed = False -# -# job, err = getJobById(jobid) -# -# if job is None: -# log.error(f"Error by requesting job with jobid {jobid}: {err['msg']}") -# return None -# -# status = job['status'] -# resourceId = job['actinia_core_jobid'] -# -# if not status or not resourceId: -# # the VM should be destroyed if the status is PREPARING and no resourcId -# # is set, e.g. if the starting of the VM fails without an ERROR -# if status == 'PREPARING' and not resourceId: -# pass -# else: -# log.error('Job status or resourceId is not set!') -# return None -# -# if resourceId: -# log.debug('Job status is ' + status + ' and resourceId is: ' + resourceId) -# url = job['actinia_core_url'] -# platform = job['actinia_core_platform'] -# -# if platform.lower() == 'vm': -# actiniacore = 'actinia-core-vm' -# vm_needs_to_be_destroyed = True -# -# if ('processing_host' in job['rule_configuration'] -# and job['rule_configuration']['processing_host'] is not None): -# vm_needs_to_be_destroyed = False -# -# if resourceId: -# connection = checkConnectionWithoutResponse(actiniacore, url) -# else: -# connection = None -# -# if status in ['PENDING', 'RUNNING'] and connection is not None: -# log.debug('Status is in PENDING or RUNNING, will cancel') -# res = cancelActiniaCore(resourceId, url, platform) -# if res: -# log.debug('Actinia-Core response TRUE') -# job = cancelJobById(jobid) -# log.debug('Job in jobtable is ' + job['status']) -# return job -# else: -# log.debug('Actinia-Core response is None') -# return None -# -# elif platform.lower() == 'vm' and vm_needs_to_be_destroyed is True: -# if connection is not None: -# log.debug('actinia-core connection exists and VM will ' + -# 'be destroyed') -# else: -# log.debug('actinia-core connection does not exist, but ' + -# 'VM will be destroyed') -# # destroy actinia-core VM -# record = destroyVM(job['process'], jobid) -# if record: -# log.debug('DestroyVM response exists') -# job = cancelJobById(jobid) -# log.debug('Job in jobtable is ' + job['status']) -# return job -# -# elif connection is not None: -# log.debug('Status not in PENDING or RUNNING and no VM to destroy, pass') -# return job -# -# else: -# log.error('There is no connection to actinia-core') -# return None diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index ea5cbec..a0fb270 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -26,36 +26,25 @@ import json import pickle -from flask import request, make_response, jsonify -from flask_restful_swagger_2 import swagger -# from flask_restful_swagger_2 import Resource -from actinia_core.models.response_models import \ - SimpleResponseModel from actinia_core.rest.base.resource_base import ResourceBase from actinia_core.core.common.redis_interface import enqueue_job -from actinia_parallel_plugin.apidocs import helloworld -from actinia_parallel_plugin.core.batches import ( - createBatch, - createBatchId, - createBatchResponseDict, - getJobsByBatchId, - # startProcessingBlock, -) +# from actinia_parallel_plugin.core.batches import ( +# # createBatch, +# # createBatchId, +# # createBatchResponseDict, +# # getJobsByBatchId, +# ) from actinia_parallel_plugin.core.jobtable import ( getJobById, ) -from actinia_parallel_plugin.model.response_models import ( - SimpleStatusCodeResponseModel, -) -# from actinia_parallel_plugin.model.batch_process_chain import ( -# SingleJob, +# from actinia_parallel_plugin.model.response_models import ( +# SimpleStatusCodeResponseModel, # ) -# from actinia_parallel_plugin.core.jobtable import updateJobByID from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log -from actinia_parallel_plugin.core.persistent_processing import start_job +# from actinia_parallel_plugin.core.persistent_processing import start_job class AsyncParallelJobResource(ResourceBase): @@ -63,75 +52,76 @@ class AsyncParallelJobResource(ResourceBase): def __init__(self, post_url, process_chain, location_name, mapset_name, batch_id, job_id): - super(AsyncParallelJobResource, self).__init__(post_url) + super(AsyncParallelJobResource, self).__init__(post_url=post_url) self.location_name = location_name self.mapset_name = mapset_name self.batch_id = batch_id self.job_id = job_id self.request_data = process_chain + self.post_url = post_url - def start_job(self, process, block): + def start_parallel_job(self, process, block): """Starting job in running actinia-core instance and update job db.""" job, err = getJobById(self.job_id) # TODO prepare_actinia ? # TODO execute_actinia ? # TODO goodby_actinia ? - # has_json = False - # self.request_data = pc - rdc = self.preprocess( has_json=False, location_name=self.location_name, mapset_name=self.mapset_name ) if rdc: - from actinia_parallel_plugin.core.persistent_processing import \ - ParallelPersistentProcessing - processing = ParallelPersistentProcessing( - rdc, self.batch_id, block, self.job_id) - # processing.run(self.request_data) - processing.run() - - # enqueue_job( - # self.job_timeout, - # start_job, - # rdc, - # self.batch_id, - # block, - # jobid, - # json.dumps(process_chain) - # ) - - # update job in jobtable to update the resource id - response_data = self.resource_logger.get( - self.user_id, self.resource_id) - http_code, response_model = pickle.loads(response_data) - job = updateJob(processing.resource_id, response_model, self.job_id) + + if process == "persistent": + from actinia_parallel_plugin.core.persistent_processing import \ + start_job + # # for debugging (works not so gogd with parallel processing) + # processing = ParallelPersistentProcessing( + # rdc, self.batch_id, block, self.job_id, self.post_url) + # processing.run() + else: + # TODO change start_job import + from actinia_parallel_plugin.core.persistent_processing import \ + start_job + enqueue_job( + self.job_timeout, + start_job, + rdc, + self.batch_id, + block, + self.job_id, + self.post_url + ) + + # update job in jobtable + html_code, response_model = pickle.loads(self.response_data) + job = updateJob(self.resource_id, response_model, self.job_id) return job def get_job_entry(self): """Return job entry by requesting jobtable from db.""" return getJobById(self.job_id)[0] - # first_jobs = self._start_processing_block(jobs_in_db, 1) - def _start_processing_block(self, jobs, block): - """Starts first processing block of jobs from batch process. - """ - jobs_to_start = [ - job for job in jobs if job["processing_block"] == block] - jobs_responses = [] - for job in jobs_to_start: - process_chain = dict() - process_chain["list"] = job["rule_configuration"]["list"] - process_chain["version"] = job["rule_configuration"]["version"] - start_kwargs = { - "process": job["process"], - "process_chain": process_chain, - "jobid": job["idpk_jobs"], - # "actinia_core_platform": job["actinia_core_platform"], - # "actinia_core_url": job["actinia_core_url"] - } - job_entry = self._start_job(**start_kwargs) - jobs_responses.append(job_entry) - return jobs_responses + # # first_jobs = self._start_processing_block(jobs_in_db, 1) + # def _start_processing_block(self, jobs, block): + # """Starts first processing block of jobs from batch process. + # """ + # jobs_to_start = [ + # job for job in jobs if job["processing_block"] == block] + # jobs_responses = [] + # for job in jobs_to_start: + # process_chain = dict() + # process_chain["list"] = job["rule_configuration"]["list"] + # process_chain["version"] = job["rule_configuration"]["version"] + # start_kwargs = { + # "process": job["process"], + # "process_chain": process_chain, + # "jobid": job["idpk_jobs"], + # # "actinia_core_platform": job["actinia_core_platform"], + # # "actinia_core_url": job["actinia_core_url"] + # } + # job_entry = self._start_job(**start_kwargs) + # jobs_responses.append(job_entry) + # return jobs_responses diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py index 397c7fd..0fedb26 100644 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -30,27 +30,31 @@ import pickle import json -from actinia_core.processing.actinia_processing.ephemeral.persistent_processing import PersistentProcessing +from actinia_core.processing.actinia_processing.ephemeral.\ + persistent_processing import PersistentProcessing from actinia_core.core.common.exceptions \ - import AsyncProcessError, AsyncProcessTermination, RsyncError + import AsyncProcessError, AsyncProcessTermination from actinia_core.core.common.exceptions import AsyncProcessTimeLimit from actinia_core.models.response_models \ - import ProcessingResponseModel, ExceptionTracebackModel + import ExceptionTracebackModel from actinia_parallel_plugin.core.batches import ( checkProcessingBlockFinished, + # createBatchResponseDict, getJobsByBatchId, + startProcessingBlock, ) from actinia_parallel_plugin.core.jobs import updateJob class ParallelPersistentProcessing(PersistentProcessing): - def __init__(self, rdc, batch_id, processing_block, jobid): + def __init__(self, rdc, batch_id, processing_block, jobid, post_url=None): super(ParallelPersistentProcessing, self).__init__(rdc) self.batch_id = batch_id self.processing_block = processing_block self.jobid = jobid + self.post_url = post_url # def _execute(self, process_chain, skip_permission_check=False): def _execute(self, skip_permission_check=False): @@ -195,8 +199,15 @@ def _update_and_check_batch_jobs(self): jobs_from_batch, block) if block_done is True and block < max(all_blocks): next_block = block + 1 - import pdb; pdb.set_trace() - print("TODO start next Block") + next_jobs = startProcessingBlock( + jobs_from_batch, + next_block, + self.batch_id, + self.location_name, + self.mapset_name, + self.post_url + ) + # print("TODO start next Block") # next_jobs = startProcessingBlock(jobs_from_batch, # next_block) # res = createBatchResponseDict(next_jobs) @@ -210,6 +221,10 @@ def _update_and_check_batch_jobs(self): def start_job(*args): - process_chain = json.loads(args[-1]) - processing = ParallelPersistentProcessing(*args[:-1]) - processing.run(process_chain) + processing = ParallelPersistentProcessing(*args) + processing.run() + +# def start_job(*args): +# process_chain = json.loads(args[-1]) +# processing = ParallelPersistentProcessing(*args[:-1]) +# processing.run(process_chain) diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index bc95373..f52a12e 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -53,13 +53,9 @@ def create_endpoints(flask_api): JobId, "/processing_parallel/jobs/") - # "/processing_parallel/jobs/" - # "/processing_parallel/batchjobs" # "/processing_parallel/jobs" - - # initilalize jobtable initJobDB() applyMigrations() diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 3b626e6..3b3829f 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -25,7 +25,7 @@ __maintainer__ = "mundialis GmbH % Co. KG" import os -import json +# import json from flask_restful_swagger_2 import Schema from actinia_core.models.process_chain import ProcessChainModel diff --git a/test_postbodies/parallel_processing.json b/test_postbodies/parallel_processing.json index 925f2ba..92383e4 100644 --- a/test_postbodies/parallel_processing.json +++ b/test_postbodies/parallel_processing.json @@ -2,9 +2,16 @@ "jobs": [ { "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block1", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ] + }, { "module": "r.mapcalc", - "id": "r_mapcalc_0_nonparallel", + "id": "r_mapcalc_0_nonparallel_block1", "inputs":[ {"param": "expression", "value": "baum = elevation@PERMANENT * 2"} ] @@ -17,7 +24,7 @@ "list": [ { "module": "g.region", - "id": "g_region_1_parallel", + "id": "g_region_1_parallel_block2", "inputs":[ {"param": "n", "value": "228500"}, {"param": "s", "value": "215000"}, @@ -27,9 +34,9 @@ }, { "module": "r.mapcalc", - "id": "r_mapcalc_1_parallel", + "id": "r_mapcalc_1_parallel_block2", "inputs":[ - {"param": "expression", "value": "baum2 = baum@test_parallel * 2"} + {"param": "expression", "value": "baum2 = baum@test * 2"} ] } ], @@ -40,7 +47,7 @@ "list": [ { "module": "g.region", - "id": "g_region_2_parallel", + "id": "g_region_2_parallel_block2", "inputs":[ {"param": "n", "value": "228500"}, {"param": "s", "value": "215000"}, @@ -50,14 +57,37 @@ }, { "module": "r.mapcalc", - "id": "r_mapcalc_2_parallel", + "id": "r_mapcalc_2_parallel_block2", "inputs":[ - {"param": "expression", "value": "baum2 = baum@test_parallel * 2"} + {"param": "expression", "value": "baum2 = baum@test * 2"} ] } ], "parallel": "true", "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block3", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ] + }, + { + "module": "r.patch", + "id": "r_patch_block3", + "inputs":[ + {"param": "input", "value": "baum2@test_parallel_1,baum2@test_parallel_2"} + ], + "outputs":[ + {"param": "input", "value": "baum2"} + ] + } + ], + "parallel": "false", + "version": "1" } ] } From d4ab5f6e72e6e8812e975e8b25eb24f9f825d8b5 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 20 May 2022 10:14:09 +0200 Subject: [PATCH 05/47] =?UTF-8?q?parallele=20processierung=20l=C3=A4uft=20?= =?UTF-8?q?mit=20fehler=20in=20processierung=20(map=20wird=20nicht=20erkan?= =?UTF-8?q?nt=20in=20r.mapcalc)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/parallel_processing.py | 28 ++- src/actinia_parallel_plugin/core/batches.py | 11 +- .../core/parallel_processing_job.py | 43 +++- .../core/parallel_resource_base.py | 230 ++++++++++++++++++ .../core/persistent_processing.py | 20 +- 5 files changed, 318 insertions(+), 14 deletions(-) create mode 100644 src/actinia_parallel_plugin/core/parallel_resource_base.py diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index ea90274..a8243cc 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -26,14 +26,19 @@ import json import pickle -from flask import request, make_response, jsonify +from flask import request, make_response, jsonify, g from flask_restful_swagger_2 import swagger # from flask_restful_swagger_2 import Resource +from actinia_api import URL_PREFIX + +# from actinia_core.core.common.app import flask_api +# from actinia_core.rest.resource_management import ResourceManager + from actinia_core.models.response_models import \ SimpleResponseModel from actinia_core.rest.base.resource_base import ResourceBase -from actinia_core.core.common.redis_interface import enqueue_job +# from actinia_core.core.common.redis_interface import enqueue_job from actinia_parallel_plugin.apidocs import helloworld from actinia_parallel_plugin.core.batches import ( @@ -170,15 +175,32 @@ def post(self, location_name, mapset_name): ))) return make_response(res, 500) + # Generate the base of the status URL + # import pdb; pdb.set_trace() + self.base_status_url = f"{request.host_url}{URL_PREFIX}/resouces/{g.user.user_id}/" + # self.base_status_url = flask_api.url_for( + # ResourceManager, + # user_id=g.user.user_id, + # resource_id="resource_id", + # _external=True + # ) + # start first processing block # first_jobs = self._start_processing_block(jobs_in_db, 1) + # import pdb; pdb.set_trace() first_jobs = startProcessingBlock( jobs_in_db, 1, self.batch_id, self.location_name, self.mapset_name, - self.post_url + g.user, + request.url, + self.post_url, + request.endpoint, + request.method, + request.path, + self.base_status_url ) first_status = [entry["status"] for entry in first_jobs] all_jobs = getJobsByBatchId(self.batch_id, "persistent") diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 0dd0f33..2439541 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -288,7 +288,8 @@ def getJobsByBatchId(batch_id, process=None): def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, - post_url): + user, request_url, post_url, endpoint, method, path, + base_status_url): """ Function to start a specific processing block for an input list of jobs (db entries) """ @@ -314,12 +315,18 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, if mapset_suffix != "": mapset_name_parallel += f"{mapset_suffix}{num}" parallel_job = AsyncParallelJobResource( + user=user, + request_url=request_url, post_url=post_url, + endpoint=endpoint, + method=method, + path=path, process_chain=process_chain, location_name=location_name, mapset_name=mapset_name_parallel, batch_id=batch_id, - job_id=jobid + job_id=jobid, + base_status_url=base_status_url ) parallel_job.start_parallel_job("persistent", 1) job_entry = parallel_job.get_job_entry() diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index a0fb270..3c0fd0a 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -27,7 +27,7 @@ import json import pickle -from actinia_core.rest.base.resource_base import ResourceBase +# from actinia_core.rest.base.resource_base import ResourceBase from actinia_core.core.common.redis_interface import enqueue_job # from actinia_parallel_plugin.core.batches import ( @@ -45,20 +45,34 @@ from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log # from actinia_parallel_plugin.core.persistent_processing import start_job +from actinia_parallel_plugin.core.parallel_resource_base import ParallelResourceBase -class AsyncParallelJobResource(ResourceBase): +class AsyncParallelJobResource(ParallelResourceBase): """Job for parallel processing""" - def __init__(self, post_url, process_chain, location_name, mapset_name, - batch_id, job_id): - super(AsyncParallelJobResource, self).__init__(post_url=post_url) + def __init__(self, user, request_url, post_url, endpoint, method, path, + process_chain, location_name, mapset_name, + batch_id, job_id, base_status_url): + super(AsyncParallelJobResource, self).__init__( + user=user, + request_url=request_url, + endpoint=endpoint, + method=method, + path=path, + post_url=post_url, + base_status_url=base_status_url + ) self.location_name = location_name self.mapset_name = mapset_name self.batch_id = batch_id self.job_id = job_id self.request_data = process_chain self.post_url = post_url + self.endpoint = endpoint + self.method = method + self.path = path + self.base_status_url = base_status_url def start_parallel_job(self, process, block): """Starting job in running actinia-core instance and update job db.""" @@ -78,8 +92,17 @@ def start_parallel_job(self, process, block): from actinia_parallel_plugin.core.persistent_processing import \ start_job # # for debugging (works not so gogd with parallel processing) + # from actinia_parallel_plugin.core.persistent_processing import \ + # ParallelPersistentProcessing # processing = ParallelPersistentProcessing( - # rdc, self.batch_id, block, self.job_id, self.post_url) + # rdc, self.batch_id, block, self.job_id, + # self.user, + # self.request_url, + # self.post_url, + # self.endpoint, + # self.method, + # self.path, + # self.base_status_url) # processing.run() else: # TODO change start_job import @@ -92,7 +115,13 @@ def start_parallel_job(self, process, block): self.batch_id, block, self.job_id, - self.post_url + self.user, + self.request_url, + self.post_url, + self.endpoint, + self.method, + self.path, + self.base_status_url ) # update job in jobtable diff --git a/src/actinia_parallel_plugin/core/parallel_resource_base.py b/src/actinia_parallel_plugin/core/parallel_resource_base.py new file mode 100644 index 0000000..fe524dd --- /dev/null +++ b/src/actinia_parallel_plugin/core/parallel_resource_base.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel processing +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import time +import os +from datetime import datetime + +from flask_restful_swagger_2 import Resource + +# from actinia_core.core.common.app import flask_api +# from actinia_core.rest.resource_streamer import RequestStreamerResource + +from actinia_core.core.common.config import global_config +from actinia_core.core.messages_logger import MessageLogger +from actinia_core.core.resources_logger import ResourceLogger +from actinia_core.rest.base.resource_base import ResourceBase +# from actinia_core.rest.resource_management import ResourceManager +from actinia_core.models.response_models import ( + create_response_from_model, + ApiInfoModel, + ProcessingResponseModel, +) +from actinia_core.core.resource_data_container import ResourceDataContainer + + +class ParallelResourceBase(ResourceBase): + """This is the base class for all asynchronous and synchronous processing + resources. + """ + + def __init__(self, user, request_url, endpoint, method, path, + base_status_url, + resource_id=None, iteration=None, post_url=None): + # Configuration + Resource.__init__(self) + + # Store the user id, user group and all credentials of the current user + self.user = user + self.user_id = user.get_id() + self.user_group = user.get_group() + self.user_role = user.get_role() + self.has_superadmin_role = user.has_superadmin_role() + self.user_credentials = user.get_credentials() + + self.orig_time = time.time() + self.orig_datetime = str(datetime.now()) + + kwargs = dict() + kwargs['host'] = global_config.REDIS_SERVER_URL + kwargs['port'] = global_config.REDIS_SERVER_PORT + if (global_config.REDIS_SERVER_PW and + global_config.REDIS_SERVER_PW is not None): + kwargs['password'] = global_config.REDIS_SERVER_PW + self.resource_logger = ResourceLogger(**kwargs) + del kwargs + + self.message_logger = MessageLogger() + + self.grass_data_base = global_config.GRASS_DATABASE + self.grass_user_data_base = global_config.GRASS_USER_DATABASE + self.grass_base_dir = global_config.GRASS_GIS_BASE + self.grass_start_script = global_config.GRASS_GIS_START_SCRIPT + self.grass_addon_path = global_config.GRASS_ADDON_PATH + self.download_cache = os.path.join( + global_config.DOWNLOAD_CACHE, self.user_id) + + # Set the resource id + if resource_id is None: + # Generate the resource id + self.request_id, self.resource_id = self.generate_uuids() + else: + self.resource_id = resource_id + self.request_id = self.generate_request_id_from_resource_id() + + # set iteration and post_url + self.iteration = iteration + self.post_url = post_url + + # The base URL's for resources that will be streamed + self.resource_url_base = None + + # Generate the status URL + # import pdb; pdb.set_trace() + self.status_url = f"{base_status_url}{self.request_id}" + # self.status_url = flask_api.url_for( + # ResourceManager, + # user_id=self.user_id, + # resource_id=self.resource_id, + # _external=True + # ) + + if global_config.FORCE_HTTPS_URLS is True and "http://" in self.status_url: + self.status_url = self.status_url.replace("http://", "https://") + + self.request_url = request_url + self.resource_url = None + self.request_data = None + self.response_data = None + self.job_timeout = 0 + + # Replace this with the correct response model in subclasses + # The class that is used to create the response + self.response_model_class = ProcessingResponseModel + + # Put API information in the response for later accounting + kwargs = { + 'endpoint': endpoint, + 'method': method, + 'path': path, + 'request_url': self.request_url} + if self.post_url is not None: + kwargs['post_url'] = self.post_url + self.api_info = ApiInfoModel(**kwargs) + + def preprocess(self, has_json=True, has_xml=False, + location_name=None, mapset_name=None, map_name=None): + """Preprocessing steps for asynchronous processing + + - Check if the request has a data field + - Check if the module chain description can be loaded + - Initialize the response and request ids as well as the + url for status polls + - Send an accept entry to the resource redis database + + Args: + has_json (bool):Set True if the request has JSON data, False otherwise + has_xml (bool):Set True if the request has XML data, False otherwise + location_name (str): The name of the location to work in + mapset_name (str): The name of the target mapset in which the + computation should be performed + map_name: The name of the map or other resource (raster, vector, + STRDS, color, ...) + + Returns: + The ResourceDataContainer that contains all required information + for the async process or None if the request was wrong. Then use + the self.response_data variable to send a response. + + """ + # if has_json is True and has_xml is True: + # if request.is_json is True: + # self.request_data = request.get_json() + # else: + # if self.check_for_xml() is False: + # return None + # elif has_xml is True: + # if self.check_for_xml() is False: + # return None + # elif has_json is True: + # if self.check_for_json() is False: + # return None + + # Compute the job timeout of the worker queue from the user credentials + process_time_limit = self.user_credentials["permissions"]["process_time_limit"] + process_num_limit = self.user_credentials["permissions"]["process_num_limit"] + self.job_timeout = int(process_time_limit * process_num_limit * 20) + + # Create the resource URL base and use a placeholder for the file name + # The placeholder __None__ must be replaced by the resource URL generator + # TODO check if this works + self.resource_url_base = f"{self.status_url}/__None__" + + if (global_config.FORCE_HTTPS_URLS is True + and "http://" in self.resource_url_base): + self.resource_url_base = self.resource_url_base.replace( + "http://", "https://") + + # Create the accepted response that will be always send + self.response_data = create_response_from_model( + self.response_model_class, + status="accepted", + user_id=self.user_id, + resource_id=self.resource_id, + iteration=self.iteration, + process_log=None, + results={}, + message="Resource accepted", + http_code=200, + orig_time=self.orig_time, + orig_datetime=self.orig_datetime, + status_url=self.status_url, + api_info=self.api_info) + + # Send the status to the database + self.resource_logger.commit( + self.user_id, self.resource_id, self.iteration, self.response_data) + + # Return the ResourceDataContainer that includes all + # required data for the asynchronous processing + return ResourceDataContainer(grass_data_base=self.grass_data_base, + grass_user_data_base=self.grass_user_data_base, + grass_base_dir=self.grass_base_dir, + request_data=self.request_data, + user_id=self.user_id, + user_group=self.user_group, + user_credentials=self.user_credentials, + resource_id=self.resource_id, + iteration=self.iteration, + status_url=self.status_url, + api_info=self.api_info, + resource_url_base=self.resource_url_base, + orig_time=self.orig_time, + orig_datetime=self.orig_datetime, + config=global_config, + location_name=location_name, + mapset_name=mapset_name, + map_name=map_name) diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py index 0fedb26..abb7633 100644 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -49,12 +49,21 @@ class ParallelPersistentProcessing(PersistentProcessing): - def __init__(self, rdc, batch_id, processing_block, jobid, post_url=None): + def __init__(self, rdc, batch_id, processing_block, jobid, + user, request_url, post_url, endpoint, method, path, + base_status_url): super(ParallelPersistentProcessing, self).__init__(rdc) self.batch_id = batch_id self.processing_block = processing_block self.jobid = jobid self.post_url = post_url + self.user = user + self.request_url = request_url + self.post_url = post_url + self.endpoint = endpoint + self.method = method + self.path = path + self.base_status_url = base_status_url # def _execute(self, process_chain, skip_permission_check=False): def _execute(self, skip_permission_check=False): @@ -199,13 +208,20 @@ def _update_and_check_batch_jobs(self): jobs_from_batch, block) if block_done is True and block < max(all_blocks): next_block = block + 1 + # import pdb; pdb.set_trace() next_jobs = startProcessingBlock( jobs_from_batch, next_block, self.batch_id, self.location_name, self.mapset_name, - self.post_url + self.user, + self.request_url, + self.post_url, + self.endpoint, + self.method, + self.path, + self.base_status_url ) # print("TODO start next Block") # next_jobs = startProcessingBlock(jobs_from_batch, From 0d3d6589ad98eeb5d35d7a5232123f84a623a6be Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 20 May 2022 13:40:06 +0200 Subject: [PATCH 06/47] ephemeral parallel processing is running but to many resources and last block is not started --- .../api/parallel_ephemeral_processing.py | 117 ++++++++++++++++++ .../api/parallel_processing.py | 8 +- src/actinia_parallel_plugin/core/batches.py | 6 +- .../core/ephemeral_processing.py | 112 +++++++++++++++++ .../core/parallel_processing_job.py | 16 +++ .../core/persistent_processing.py | 10 +- src/actinia_parallel_plugin/endpoints.py | 10 ++ .../parallel_ephemeral_processing.json | 80 ++++++++++++ 8 files changed, 344 insertions(+), 15 deletions(-) create mode 100644 src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py create mode 100644 src/actinia_parallel_plugin/core/ephemeral_processing.py create mode 100644 test_postbodies/parallel_ephemeral_processing.json diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py new file mode 100644 index 0000000..d22ba4a --- /dev/null +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018-present mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel processing +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +from flask import request, make_response, jsonify, g +from flask_restful_swagger_2 import swagger + +from actinia_api import URL_PREFIX + +from actinia_core.models.response_models import \ + SimpleResponseModel +from actinia_core.rest.base.resource_base import ResourceBase + +from actinia_parallel_plugin.apidocs import helloworld +from actinia_parallel_plugin.core.batches import ( + createBatch, + createBatchId, + createBatchResponseDict, + getJobsByBatchId, + startProcessingBlock, +) + +from actinia_parallel_plugin.resources.logging import log + + +class AsyncParallelEphermeralResource(ResourceBase): + """Resource for parallel processing""" + + def __init__(self): + super(AsyncParallelEphermeralResource, self).__init__() + self.location_name = None + self.batch_id = None + + + # TODO change apidocs + @swagger.doc(helloworld.describeHelloWorld_get_docs) + # def get(self): + def post(self, location_name): + """Persistent parallel processing.""" + + self.location_name = location_name + self.post_url = self.api_info["request_url"] + + json_dict = request.get_json(force=True) + log.info("Received HTTP POST with batchjob: %s" % + str(json_dict)) + + # assign new batchid + self.batch_id = createBatchId() + # create processing blocks and insert jobs into jobtable + jobs_in_db = createBatch(json_dict, "ephemeral", self.batch_id) + if jobs_in_db is None: + res = (jsonify(SimpleResponseModel( + status=500, + message=('Error: Batch Processing Chain JSON has no ' + 'jobs.') + ))) + return make_response(res, 500) + + # Generate the base of the status URL + # import pdb; pdb.set_trace() + self.base_status_url = f"{request.host_url}{URL_PREFIX}/resouces/" \ + f"{g.user.user_id}/" + + # start first processing block + first_jobs = startProcessingBlock( + jobs_in_db, + 1, + self.batch_id, + self.location_name, + None, # mapset_name + g.user, + request.url, + self.post_url, + request.endpoint, + request.method, + request.path, + self.base_status_url, + "ephemeral" + ) + first_status = [entry["status"] for entry in first_jobs] + all_jobs = getJobsByBatchId(self.batch_id, "ephemeral") + if None in first_jobs: + res = (jsonify(SimpleResponseModel( + status=500, + message=('Error: There was a problem starting the ' + 'first jobs of the batchjob.') + ))) + return make_response(res, 500) + elif "ERROR" not in first_status: + return make_response(jsonify(createBatchResponseDict(all_jobs)), + 201) + else: + return make_response(jsonify(createBatchResponseDict(all_jobs)), + 412) diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index a8243cc..d0fcd00 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -24,8 +24,7 @@ __copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" -import json -import pickle + from flask import request, make_response, jsonify, g from flask_restful_swagger_2 import swagger # from flask_restful_swagger_2 import Resource @@ -193,14 +192,15 @@ def post(self, location_name, mapset_name): 1, self.batch_id, self.location_name, - self.mapset_name, g.user, request.url, self.post_url, request.endpoint, request.method, request.path, - self.base_status_url + self.base_status_url, + self.mapset_name, + "persistent" ) first_status = [entry["status"] for entry in first_jobs] all_jobs = getJobsByBatchId(self.batch_id, "persistent") diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 2439541..a9a17aa 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -289,7 +289,7 @@ def getJobsByBatchId(batch_id, process=None): def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, user, request_url, post_url, endpoint, method, path, - base_status_url): + base_status_url, process): """ Function to start a specific processing block for an input list of jobs (db entries) """ @@ -312,7 +312,7 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, # "actinia_core_url": job["actinia_core_url"] } mapset_name_parallel = mapset_name - if mapset_suffix != "": + if mapset_suffix != "" and mapset_name is not None: mapset_name_parallel += f"{mapset_suffix}{num}" parallel_job = AsyncParallelJobResource( user=user, @@ -328,7 +328,7 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, job_id=jobid, base_status_url=base_status_url ) - parallel_job.start_parallel_job("persistent", 1) + parallel_job.start_parallel_job(process, 1) job_entry = parallel_job.get_job_entry() jobs_responses.append(job_entry) return jobs_responses diff --git a/src/actinia_parallel_plugin/core/ephemeral_processing.py b/src/actinia_parallel_plugin/core/ephemeral_processing.py new file mode 100644 index 0000000..4e62371 --- /dev/null +++ b/src/actinia_parallel_plugin/core/ephemeral_processing.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel ephemeral processing +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +import pickle + +from actinia_core.processing.actinia_processing.ephemeral_processing import \ + EphemeralProcessing + +from actinia_parallel_plugin.core.batches import ( + checkProcessingBlockFinished, + getJobsByBatchId, + startProcessingBlock, +) +from actinia_parallel_plugin.core.jobs import updateJob + + +class ParallelEphemeralProcessing(EphemeralProcessing): + + def __init__(self, rdc, batch_id, processing_block, jobid, + user, request_url, post_url, endpoint, method, path, + base_status_url): + super(ParallelEphemeralProcessing, self).__init__(rdc) + self.batch_id = batch_id + self.processing_block = processing_block + self.jobid = jobid + self.post_url = post_url + self.user = user + self.request_url = request_url + self.post_url = post_url + self.endpoint = endpoint + self.method = method + self.path = path + self.base_status_url = base_status_url + + def run(self): + super(ParallelEphemeralProcessing, self).run() + self._update_and_check_batch_jobs() + + def _update_and_check_batch_jobs(self): + """Checks batch jobs and starts new batch block if the current block + is successfully finished. + """ + + # update job to finished + resource_id = self.resource_id + response_data = self.resource_logger.get( + self.user_id, self.resource_id) + http_code, response_model = pickle.loads(response_data) + updateJob(resource_id, response_model, self.jobid) + + if "finished" == response_model["status"]: + jobs_from_batch = getJobsByBatchId( + self.batch_id, + "ephemeral" + ) + all_blocks = [ + job["processing_block"] for job in jobs_from_batch] + block = int(self.processing_block) + block_done = checkProcessingBlockFinished( + jobs_from_batch, block) + if block_done is True and block < max(all_blocks): + next_block = block + 1 + next_jobs = startProcessingBlock( + jobs_from_batch, + next_block, + self.batch_id, + self.location_name, + None, # mapset_name + self.user, + self.request_url, + self.post_url, + self.endpoint, + self.method, + self.path, + self.base_status_url, + "ephemeral" + ) + + elif (response_model["status"] == "error" or + response_model["status"] == "terminated"): + # In this case, nothing happens and the next block is not + # started. + pass + + +def start_job(*args): + processing = ParallelEphemeralProcessing(*args) + processing.run() diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index 3c0fd0a..e9d5f0c 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -104,6 +104,22 @@ def start_parallel_job(self, process, block): # self.path, # self.base_status_url) # processing.run() + elif process == "ephemeral": + from actinia_parallel_plugin.core.ephemeral_processing import \ + start_job + # # for debugging (works not so gogd with parallel processing) + # from actinia_parallel_plugin.core.ephemeral_processing import \ + # ParallelEphemeralProcessing + # processing = ParallelEphemeralProcessing( + # rdc, self.batch_id, block, self.job_id, + # self.user, + # self.request_url, + # self.post_url, + # self.endpoint, + # self.method, + # self.path, + # self.base_status_url) + # processing.run() else: # TODO change start_job import from actinia_parallel_plugin.core.persistent_processing import \ diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py index abb7633..5ecd90f 100644 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -28,7 +28,6 @@ import sys import traceback import pickle -import json from actinia_core.processing.actinia_processing.ephemeral.\ persistent_processing import PersistentProcessing @@ -208,7 +207,6 @@ def _update_and_check_batch_jobs(self): jobs_from_batch, block) if block_done is True and block < max(all_blocks): next_block = block + 1 - # import pdb; pdb.set_trace() next_jobs = startProcessingBlock( jobs_from_batch, next_block, @@ -221,13 +219,9 @@ def _update_and_check_batch_jobs(self): self.endpoint, self.method, self.path, - self.base_status_url + self.base_status_url, + "persistent" ) - # print("TODO start next Block") - # next_jobs = startProcessingBlock(jobs_from_batch, - # next_block) - # res = createBatchResponseDict(next_jobs) - # return make_response(jsonify(res), 200) elif (response_model["status"] == "error" or response_model["status"] == "terminated"): diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index f52a12e..f65933f 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -29,6 +29,8 @@ from actinia_parallel_plugin.api.job import JobId from actinia_parallel_plugin.api.parallel_processing import \ AsyncParallelPersistentResource +from actinia_parallel_plugin.api.parallel_ephemeral_processing import \ + AsyncParallelEphermeralResource from actinia_parallel_plugin.core.jobtable import initJobDB, applyMigrations @@ -37,6 +39,14 @@ def create_endpoints(flask_api): apidoc = flask_api + + # POST parallel ephemeral processing + apidoc.add_resource( + AsyncParallelEphermeralResource, + "/locations//processing_parallel") + + + # POST parallel persistent processing apidoc.add_resource( AsyncParallelPersistentResource, "/locations//mapsets/" diff --git a/test_postbodies/parallel_ephemeral_processing.json b/test_postbodies/parallel_ephemeral_processing.json new file mode 100644 index 0000000..3b08057 --- /dev/null +++ b/test_postbodies/parallel_ephemeral_processing.json @@ -0,0 +1,80 @@ +{ + "jobs": [ + { + "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block1", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ] + }, + { + "module": "r.mapcalc", + "id": "r_mapcalc_0_nonparallel_block1", + "inputs":[ + {"param": "expression", "value": "baum = elevation@PERMANENT * 2"} + ] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_1_parallel_block2", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "flags": "p" + }, + { + "module": "r.info", + "id": "r_info_1_parallel_block2", + "inputs":[ + {"param": "map", "value": "elevation@PERMANENT"} + ] + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_2_parallel_block2", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "flags": "p" + }, + { + "module": "r.univar", + "id": "r_univar_2_parallel_block2", + "inputs":[ + {"param": "map", "value": "elevation@PERMANENT"} + ] + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block3", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "flags": "p" + } + ], + "parallel": "false", + "version": "1" + } + ] +} From 0b8f29d6d6b6910aaa744e8499c03cfc18ec2cac Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Mon, 23 May 2022 13:00:02 +0200 Subject: [PATCH 07/47] processing working sync (non parallelt) --- .../api/parallel_ephemeral_processing.py | 40 ++++++++--- .../api/parallel_processing.py | 11 ++-- src/actinia_parallel_plugin/core/batches.py | 2 +- .../core/ephemeral_processing.py | 2 +- .../core/parallel_processing_job.py | 66 +++++++++++-------- .../core/parallel_resource_base.py | 9 +-- 6 files changed, 76 insertions(+), 54 deletions(-) diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index d22ba4a..d2723b2 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -25,13 +25,18 @@ __maintainer__ = "mundialis GmbH % Co. KG" from flask import request, make_response, jsonify, g -from flask_restful_swagger_2 import swagger +from flask_restful_swagger_2 import swagger, Resource from actinia_api import URL_PREFIX from actinia_core.models.response_models import \ SimpleResponseModel -from actinia_core.rest.base.resource_base import ResourceBase +from actinia_core.core.common.config import global_config +from actinia_core.core.common.api_logger import log_api_call +from actinia_core.rest.base.user_auth import check_user_permissions +from actinia_core.rest.base.user_auth import create_dummy_user +from actinia_core.core.common.app import auth +# from actinia_core.rest.base.resource_base import ResourceBase from actinia_parallel_plugin.apidocs import helloworld from actinia_parallel_plugin.core.batches import ( @@ -45,15 +50,27 @@ from actinia_parallel_plugin.resources.logging import log -class AsyncParallelEphermeralResource(ResourceBase): +class AsyncParallelEphermeralResource(Resource): """Resource for parallel processing""" + decorators = [] + + # if global_config.LOG_API_CALL is True: + # decorators.append(log_api_call) + # + # if global_config.CHECK_CREDENTIALS is True: + # decorators.append(check_user_permissions) + + if global_config.LOGIN_REQUIRED is True: + decorators.append(auth.login_required) + else: + decorators.append(create_dummy_user) + def __init__(self): super(AsyncParallelEphermeralResource, self).__init__() self.location_name = None self.batch_id = None - # TODO change apidocs @swagger.doc(helloworld.describeHelloWorld_get_docs) # def get(self): @@ -61,7 +78,7 @@ def post(self, location_name): """Persistent parallel processing.""" self.location_name = location_name - self.post_url = self.api_info["request_url"] + self.post_url = request.base_url json_dict = request.get_json(force=True) log.info("Received HTTP POST with batchjob: %s" % @@ -80,9 +97,16 @@ def post(self, location_name): return make_response(res, 500) # Generate the base of the status URL - # import pdb; pdb.set_trace() - self.base_status_url = f"{request.host_url}{URL_PREFIX}/resouces/" \ - f"{g.user.user_id}/" + host_url = request.host_url + if host_url.endswith("/") and URL_PREFIX.startswith("/"): + self.base_status_url = f"{host_url[:-1]}{URL_PREFIX}/" \ + f"resources/{g.user.user_id}/" + elif not host_url.endswith("/") and not URL_PREFIX.startswith("/"): + self.base_status_url = f"{host_url}/{URL_PREFIX}/resources/" \ + f"{g.user.user_id}/" + else: + self.base_status_url = f"{host_url}{URL_PREFIX}/resources/" \ + f"{g.user.user_id}/" # start first processing block first_jobs = startProcessingBlock( diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index d0fcd00..ff4ba81 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -26,8 +26,7 @@ from flask import request, make_response, jsonify, g -from flask_restful_swagger_2 import swagger -# from flask_restful_swagger_2 import Resource +from flask_restful_swagger_2 import swagger, Resource from actinia_api import URL_PREFIX @@ -36,7 +35,7 @@ from actinia_core.models.response_models import \ SimpleResponseModel -from actinia_core.rest.base.resource_base import ResourceBase +# from actinia_core.rest.base.resource_base import ResourceBase # from actinia_core.core.common.redis_interface import enqueue_job from actinia_parallel_plugin.apidocs import helloworld @@ -63,7 +62,7 @@ # from actinia_parallel_plugin.core.batches import startProcessingBlock -class AsyncParallelPersistentResource(ResourceBase): +class AsyncParallelPersistentResource(Resource): """Resource for parallel processing""" def __init__(self): @@ -175,8 +174,7 @@ def post(self, location_name, mapset_name): return make_response(res, 500) # Generate the base of the status URL - # import pdb; pdb.set_trace() - self.base_status_url = f"{request.host_url}{URL_PREFIX}/resouces/{g.user.user_id}/" + self.base_status_url = f"{request.host_url}{URL_PREFIX}/resources/{g.user.user_id}/" # self.base_status_url = flask_api.url_for( # ResourceManager, # user_id=g.user.user_id, @@ -186,7 +184,6 @@ def post(self, location_name, mapset_name): # start first processing block # first_jobs = self._start_processing_block(jobs_in_db, 1) - # import pdb; pdb.set_trace() first_jobs = startProcessingBlock( jobs_in_db, 1, diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index a9a17aa..b154ea9 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -328,7 +328,7 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, job_id=jobid, base_status_url=base_status_url ) - parallel_job.start_parallel_job(process, 1) + parallel_job.start_parallel_job(process, block) job_entry = parallel_job.get_job_entry() jobs_responses.append(job_entry) return jobs_responses diff --git a/src/actinia_parallel_plugin/core/ephemeral_processing.py b/src/actinia_parallel_plugin/core/ephemeral_processing.py index 4e62371..57add45 100644 --- a/src/actinia_parallel_plugin/core/ephemeral_processing.py +++ b/src/actinia_parallel_plugin/core/ephemeral_processing.py @@ -69,7 +69,7 @@ def _update_and_check_batch_jobs(self): resource_id = self.resource_id response_data = self.resource_logger.get( self.user_id, self.resource_id) - http_code, response_model = pickle.loads(response_data) + _, response_model = pickle.loads(response_data) updateJob(resource_id, response_model, self.jobid) if "finished" == response_model["status"]: diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index e9d5f0c..24e3b22 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -107,41 +107,49 @@ def start_parallel_job(self, process, block): elif process == "ephemeral": from actinia_parallel_plugin.core.ephemeral_processing import \ start_job - # # for debugging (works not so gogd with parallel processing) - # from actinia_parallel_plugin.core.ephemeral_processing import \ - # ParallelEphemeralProcessing - # processing = ParallelEphemeralProcessing( - # rdc, self.batch_id, block, self.job_id, - # self.user, - # self.request_url, - # self.post_url, - # self.endpoint, - # self.method, - # self.path, - # self.base_status_url) - # processing.run() + # for debugging + for var in [ + 'GISRC', 'GISBASE', 'LD_LIBRARY_PATH', + 'GRASS_ADDON_PATH', 'GIS_LOCK']: + import os + if var in os.environ: + del os.environ[var] + from actinia_parallel_plugin.core.ephemeral_processing import \ + ParallelEphemeralProcessing + processing = ParallelEphemeralProcessing( + rdc, self.batch_id, block, self.job_id, + self.user, + self.request_url, + self.post_url, + self.endpoint, + self.method, + self.path, + self.base_status_url) + processing.run() else: # TODO change start_job import from actinia_parallel_plugin.core.persistent_processing import \ start_job - enqueue_job( - self.job_timeout, - start_job, - rdc, - self.batch_id, - block, - self.job_id, - self.user, - self.request_url, - self.post_url, - self.endpoint, - self.method, - self.path, - self.base_status_url - ) + # enqueue_job( + # self.job_timeout, + # start_job, + # rdc, + # self.batch_id, + # block, + # self.job_id, + # self.user, + # self.request_url, + # self.post_url, + # self.endpoint, + # self.method, + # self.path, + # self.base_status_url + # ) # update job in jobtable - html_code, response_model = pickle.loads(self.response_data) + self.response_data = self.resource_logger.get( + self.user_id, self.resource_id, self.iteration) + _, response_model = pickle.loads(self.response_data) job = updateJob(self.resource_id, response_model, self.job_id) return job diff --git a/src/actinia_parallel_plugin/core/parallel_resource_base.py b/src/actinia_parallel_plugin/core/parallel_resource_base.py index fe524dd..cddcbe2 100644 --- a/src/actinia_parallel_plugin/core/parallel_resource_base.py +++ b/src/actinia_parallel_plugin/core/parallel_resource_base.py @@ -103,14 +103,7 @@ def __init__(self, user, request_url, endpoint, method, path, self.resource_url_base = None # Generate the status URL - # import pdb; pdb.set_trace() - self.status_url = f"{base_status_url}{self.request_id}" - # self.status_url = flask_api.url_for( - # ResourceManager, - # user_id=self.user_id, - # resource_id=self.resource_id, - # _external=True - # ) + self.status_url = f"{base_status_url}{self.resource_id}" if global_config.FORCE_HTTPS_URLS is True and "http://" in self.status_url: self.status_url = self.status_url.replace("http://", "https://") From c806144386732f2d11ad9ebf8d9b801a90295d60 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Tue, 24 May 2022 21:32:22 +0200 Subject: [PATCH 08/47] working ephemeral processing --- .flake8 | 3 +- README.md | 54 +- .../actinia-parallel-plugin-test.cfg | 2 +- docker/actinia.cfg | 2 +- requirements.txt | 1 + setup.cfg | 5 +- .../api/parallel_ephemeral_processing.py | 15 +- .../api/parallel_processing.py | 99 +--- src/actinia_parallel_plugin/apidocs/batch.py | 40 ++ .../batchjob_post_response_example.json | 301 ++++------ .../jobs_get_docs_response_example.json | 539 ++++++++++++++++++ .../apidocs/helloworld.py | 69 --- src/actinia_parallel_plugin/apidocs/jobs.py | 187 +----- .../apidocs/regeldatei.py | 96 +--- src/actinia_parallel_plugin/core/batches.py | 21 +- .../core/ephemeral_processing.py | 2 +- src/actinia_parallel_plugin/core/jobs.py | 5 +- src/actinia_parallel_plugin/core/jobtable.py | 13 +- .../core/parallel_processing_job.py | 136 ++--- .../core/parallel_resource_base.py | 72 ++- .../core/persistent_processing.py | 16 +- src/actinia_parallel_plugin/endpoints.py | 17 +- src/actinia_parallel_plugin/model/batch.py | 33 +- .../model/jobtabelle.py | 4 +- .../resources/config.py | 2 +- .../resources/logging.py | 8 +- .../parallel_ephemeral_processing.json | 16 +- 27 files changed, 956 insertions(+), 802 deletions(-) create mode 100644 src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json delete mode 100644 src/actinia_parallel_plugin/apidocs/helloworld.py diff --git a/.flake8 b/.flake8 index 0baa625..7462d1d 100644 --- a/.flake8 +++ b/.flake8 @@ -7,4 +7,5 @@ exclude = .git,.pycache,build,.eggs per-file-ignores = ./src/actinia_parallel_plugin/wsgi.py: F401 - ./tests/test_resource_base.py: F401 + ./tests/test_resource_base.py: F401 + ./src/actinia_parallel_plugin/core/persistent_processing.py: E501 diff --git a/README.md b/README.md index 3047006..732b310 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # actinia-parallel-plugin -This is an example plugin for [actinia-core](https://github.com/mundialis/actinia_core) which adds a "Hello World" endpoint to actinia-core. +This is the actinia-parallel-plugin for [actinia-core](https://github.com/mundialis/actinia_core) which adds parallel processing endpoints to actinia. You can run actinia-parallel-plugin as an actinia-core plugin. @@ -20,14 +20,6 @@ docker network prune docker-compose -f docker/docker-compose.yml up -d ``` -### Requesting helloworld endpoint -You can test the plugin and request the `/helloworld` endpoint, e.g. with: -``` -curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel | jq - -curl -u actinia-gdi:actinia-gdi -H 'accept: application/json' -H 'Content-Type: application/json' -X POST http://localhost:8088/api/v3/processing_parallel -d '{"name": "test"}' | jq -``` - ## DEV setup For a DEV setup you can use the docker/docker-compose.yml: ``` @@ -38,6 +30,9 @@ docker-compose -f docker/docker-compose.yml run --rm --service-ports --entrypoin (cd /src/actinia-parallel-plugin && python3 setup.py install) # start actinia-core with your plugin gunicorn -b 0.0.0.0:8088 -w 1 --access-logfile=- -k gthread actinia_core.main:flask_app + +# or for debugging in one line with reset +reset && (cd /src/actinia-parallel-plugin && python3 setup.py install) && gunicorn -b 0.0.0.0:8088 -w 3 --access-logfile=- -k gthread actinia_core.main:flask_app ``` ### Postgis @@ -83,4 +78,43 @@ make integrationtest make devtest ``` -## +## Examples + +### Requesting batch job and job endpoints +``` +# request batch job +curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel/batchjobs/1 | jq +# request job +curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel/jobs/1 | jq +``` + +### Start parallel batch job +#### Ephemeral processing +You can start a parallel **ephemeral** batch job via: +``` +# parallel ephemeral processing +curl -u actinia-gdi:actinia-gdi -X POST -H 'Content-Type: application/json' -d @test_postbodies/parallel_ephemeral_processing.json http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel | jq +``` +Attention: +* The individual process chains must be "independent" of each other, since + createBatch is designed as an ephemeral process. + +TODOs: +* Test of exporters in PC +* APIDOCS +* Tests +* using stdout/export in PC of next block + +#### Persistent processing +You can also start a **persistent** batch job via: +``` +# parallel persistent processing (TODO!!!) +curl -u actinia-gdi:actinia-gdi -X POST -H 'Content-Type: application/json' -d @test_postbodies/parallel_processing.json http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/mapsets/test_mapset/processing_parallel | jq +``` +Hereby, the parallel started process chains will be computed in mapsets with +the suffix `_parallel_{NUMBER}` (see teh example process chains in +`test_postbodies/parallel_persistent_processing.json`). +So if you want to use a mapset in a later step, you have to pay attention to +the naming of the mapset. + +TODOs: alles diff --git a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg index 05adc95..7dde08a 100644 --- a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg +++ b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg @@ -3,7 +3,7 @@ grass_database = /actinia_core/grassdb grass_user_database = /actinia_core/userdata grass_tmp_database = /actinia_core/workspace/temp_db grass_resource_dir = /actinia_core/resources -grass_gis_base = /usr/local/grass80 +grass_gis_base = /usr/local/grass grass_gis_start_script = /usr/local/bin/grass grass_addon_path = /root/.grass8/addons/ diff --git a/docker/actinia.cfg b/docker/actinia.cfg index 0baab7c..3fa6046 100644 --- a/docker/actinia.cfg +++ b/docker/actinia.cfg @@ -3,7 +3,7 @@ grass_database = /actinia_core/grassdb grass_user_database = /actinia_core/userdata grass_tmp_database = /actinia_core/workspace/temp_db grass_resource_dir = /actinia_core/resources -grass_gis_base = /usr/local/grass80 +grass_gis_base = /usr/local/grass grass_gis_start_script = /usr/local/bin/grass grass_addon_path = /root/.grass8/addons/ diff --git a/requirements.txt b/requirements.txt index 95754a4..3483f5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ python-json-logger==0.1.11 peewee yoyo-migrations psycopg2 +jsonmodels diff --git a/setup.cfg b/setup.cfg index e1b837e..6e25fbc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ name = actinia_parallel_plugin.wsgi description = actinia example plugin author = Anika Weinmann author-email = aweinmann@mundialis.de -license = mit +license = GPU3 long-description = file: README.md long-description-content-type = text/x-rst; charset=UTF-8 url = https://github.com/pyscaffold/pyscaffold/ @@ -36,6 +36,9 @@ setup_requires = pyscaffold>=3.2a0,<3.3a0 # Require a specific Python version, e.g. Python 2.7 or >= 3.4 # python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* +[options.package_data] +* = **/*.json + [options.packages.find] where = src exclude = diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index d2723b2..35d7c43 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -29,16 +29,15 @@ from actinia_api import URL_PREFIX +from actinia_core.core.common.app import auth +from actinia_core.core.common.config import global_config +# from actinia_core.core.common.api_logger import log_api_call from actinia_core.models.response_models import \ SimpleResponseModel -from actinia_core.core.common.config import global_config -from actinia_core.core.common.api_logger import log_api_call -from actinia_core.rest.base.user_auth import check_user_permissions +# from actinia_core.rest.base.user_auth import check_user_permissions from actinia_core.rest.base.user_auth import create_dummy_user -from actinia_core.core.common.app import auth -# from actinia_core.rest.base.resource_base import ResourceBase -from actinia_parallel_plugin.apidocs import helloworld +from actinia_parallel_plugin.apidocs import batch from actinia_parallel_plugin.core.batches import ( createBatch, createBatchId, @@ -46,7 +45,6 @@ getJobsByBatchId, startProcessingBlock, ) - from actinia_parallel_plugin.resources.logging import log @@ -71,8 +69,7 @@ def __init__(self): self.location_name = None self.batch_id = None - # TODO change apidocs - @swagger.doc(helloworld.describeHelloWorld_get_docs) + @swagger.doc(batch.batchjobs_post_docs) # def get(self): def post(self, location_name): """Persistent parallel processing.""" diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py index ff4ba81..5b57863 100644 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_processing.py @@ -24,19 +24,12 @@ __copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" - from flask import request, make_response, jsonify, g from flask_restful_swagger_2 import swagger, Resource from actinia_api import URL_PREFIX - -# from actinia_core.core.common.app import flask_api -# from actinia_core.rest.resource_management import ResourceManager - from actinia_core.models.response_models import \ SimpleResponseModel -# from actinia_core.rest.base.resource_base import ResourceBase -# from actinia_core.core.common.redis_interface import enqueue_job from actinia_parallel_plugin.apidocs import helloworld from actinia_parallel_plugin.core.batches import ( @@ -46,20 +39,10 @@ getJobsByBatchId, startProcessingBlock, ) -# from actinia_parallel_plugin.core.jobtable import ( -# getJobById, -# ) from actinia_parallel_plugin.model.response_models import ( SimpleStatusCodeResponseModel, ) -# from actinia_parallel_plugin.model.batch_process_chain import ( -# SingleJob, -# ) -# from actinia_parallel_plugin.core.jobtable import updateJobByID -# from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log -# from actinia_parallel_plugin.core.persistent_processing import start_job -# from actinia_parallel_plugin.core.batches import startProcessingBlock class AsyncParallelPersistentResource(Resource): @@ -76,80 +59,8 @@ def get(self, location_name, mapset_name): """Get 'Hello world!' as answer string.""" return SimpleStatusCodeResponseModel(status=200, message="TEST") - # def prepare_actinia(self): - # e.g. start a VM and check connection to actinia-core on it - # return things - - # def _start_job(self, process, process_chain, jobid): - # """Starting job in running actinia-core instance and update job db.""" - # job, err = getJobById(jobid) - # # TODO prepare_actinia ? - # # TODO execute_actinia ? - # # TODO goodby_actinia ? - # - # # has_json = False - # # self.request_data = pc - # - # rdc = self.preprocess( - # has_json=True, - # location_name=self.location_name, - # mapset_name=self.mapset_name - # ) - # if rdc: - # block = 1 - # from actinia_parallel_plugin.core.persistent_processing import \ - # ParallelPersistentProcessing - # processing = ParallelPersistentProcessing( - # rdc, self.batch_id, block, jobid) - # processing.run(process_chain) - # - # # enqueue_job( - # # self.job_timeout, - # # start_job, - # # rdc, - # # self.batch_id, - # # block, - # # jobid, - # # json.dumps(process_chain) - # # ) - # - # job = getJobById(jobid)[0] - # return job - - # def _start_processing_block(self, jobs, block): - # """Starts first processing block of jobs from batch process. - # """ - # jobs_to_start = [ - # job for job in jobs if job["processing_block"] == block] - # jobs_responses = [] - # for job in jobs_to_start: - # process_chain = dict() - # process_chain["list"] = job["rule_configuration"]["list"] - # process_chain["version"] = job["rule_configuration"]["version"] - # jobid = job["idpk_jobs"] - # start_kwargs = { - # "process": job["process"], - # "process_chain": process_chain, - # "jobid": job["idpk_jobs"], - # # "actinia_core_platform": job["actinia_core_platform"], - # # "actinia_core_url": job["actinia_core_url"] - # } - # parallel_job = AsyncParallelJobResource( - # post_url=self.post_url, - # process_chain=process_chain, - # location_name=self.location_name, - # mapset_name=self.mapset_name, - # batch_id=self.batch_id, - # job_id=jobid - # ) - # parallel_job.start_job("persistent", 1) - # job_entry = parallel_job.get_job_entry() - # jobs_responses.append(job_entry) - # return jobs_responses - # TODO get all batch jobs @swagger.doc(helloworld.describeHelloWorld_get_docs) - # def get(self): def post(self, location_name, mapset_name): """Persistent parallel processing.""" @@ -174,16 +85,10 @@ def post(self, location_name, mapset_name): return make_response(res, 500) # Generate the base of the status URL - self.base_status_url = f"{request.host_url}{URL_PREFIX}/resources/{g.user.user_id}/" - # self.base_status_url = flask_api.url_for( - # ResourceManager, - # user_id=g.user.user_id, - # resource_id="resource_id", - # _external=True - # ) + self.base_status_url = f"{request.host_url}{URL_PREFIX}/resources/" \ + f"{g.user.user_id}/" # start first processing block - # first_jobs = self._start_processing_block(jobs_in_db, 1) first_jobs = startProcessingBlock( jobs_in_db, 1, diff --git a/src/actinia_parallel_plugin/apidocs/batch.py b/src/actinia_parallel_plugin/apidocs/batch.py index 4435190..a83d026 100644 --- a/src/actinia_parallel_plugin/apidocs/batch.py +++ b/src/actinia_parallel_plugin/apidocs/batch.py @@ -30,6 +30,7 @@ from actinia_parallel_plugin.model.batch import ( BatchJobResponseModel, + BatchProcessChainModel, ) @@ -66,3 +67,42 @@ } } } + +batchjobs_post_docs = { + "summary": "Creates a new Batchjob from a Batch Processing Chain.", + "description": ("This request will read the json object," + " break it up into parallel processing blocks," + " create individual jobs in the jobtable, " + " and start the jobs in actinia-core depending on" + " their processing block."), + "tags": [ + "processing" + ], + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Batch Processing Chain as json object", + "required": True, + "schema": BatchProcessChainModel + } + ], + "responses": { + "201": { + "description": ("The batchjob summary of the created batchjob and " + "all corresponding jobs"), + "schema": BatchJobResponseModel + }, + "412": { + "description": ("The batchjob summary of the created batchjob and " + "all corresponding jobs in case a job responded " + "with an error"), + "schema": BatchJobResponseModel + }, + "500": { + "description": ("The error message and a detailed log why " + "creating a batchjob failed"), + "schema": SimpleResponseModel + } + } +} diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index 4aeb485..bf30167 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -1,175 +1,91 @@ { "actinia_core_jobid": [ + "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", "None", - "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", - "resource_id-7565f55f-b798-417e-8441-e148df6194d4" + "None", + "None" ], - "actinia_core_platform": "TODO", - "actinia_core_platform_name": "example_name", + "actinia_core_platform": null, + "actinia_core_platform_name": null, "actinia_core_response": { - "resource_id-7565f55f-b798-417e-8441-e148df6194d4": { - "accept_datetime": "2021-11-26 09:52:54.961197", - "accept_timestamp": 1637920374.9611967, + "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18": { + "accept_datetime": "2022-05-24 18:22:42.343953", + "accept_timestamp": 1653416562.3439503, "api_info": { - "endpoint": "asyncephemeralexportresource", + "endpoint": "asyncparallelephermeralresource", "method": "POST", - "path": "/api/v1/locations/utm32n/processing_async_export", - "request_url": "http://actinia-core-docker:8088/api/v1/locations/utm32n/processing_async_export" + "path": "/api/v3/locations/nc_spm_08_grass7_root/processing_parallel", + "post_url": "http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel", + "request_url": "http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel" }, - "datetime": "2021-11-26 09:52:54.962316", + "datetime": "2022-05-24 18:22:42.346947", "http_code": 200, "message": "Resource accepted", "process_chain_list": [], "process_results": {}, - "resource_id": "resource_id-7565f55f-b798-417e-8441-e148df6194d4", + "resource_id": "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", "status": "accepted", - "time_delta": 0.001125335693359375, - "timestamp": 1637920374.9623156, + "time_delta": 0.0030062198638916016, + "timestamp": 1653416562.346946, "urls": { "resources": [], - "status": "http://actinia-core-docker:8088/api/v1/resources/actinia-gdi/resource_id-7565f55f-b798-417e-8441-e148df6194d4" - }, - "user_id": "actinia-gdi" - }, - "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2": { - "accept_datetime": "2021-11-26 09:52:54.610856", - "accept_timestamp": 1637920374.6108556, - "api_info": { - "endpoint": "asyncephemeralexportresource", - "method": "POST", - "path": "/api/v1/locations/utm32n/processing_async_export", - "request_url": "http://actinia-core-docker:8088/api/v1/locations/utm32n/processing_async_export" - }, - "datetime": "2021-11-26 09:52:54.782627", - "http_code": 200, - "message": "Processing successfully finished", - "process_chain_list": [ - { - "list": [ - { - "flags": "p", - "id": "g_region_test_0", - "inputs": [ - { - "param": "w", - "value": "366166" - }, - { - "param": "s", - "value": "5628872" - }, - { - "param": "e", - "value": "409135" - }, - { - "param": "n", - "value": "5681310" - }, - { - "param": "res", - "value": "10" - } - ], - "module": "g.region" - } - ], - "version": "1", - "webhooks": { - "finished": "http://actinia-gdi:5000/resources/processes/operations/update", - "update": "http://actinia-gdi:5000/resources/processes/operations/update" - } - } - ], - "process_log": [ - { - "executable": "g.region", - "id": "g_region_test_0", - "mapset_size": 429, - "parameter": [ - "w=366166", - "s=5628872", - "e=409135", - "n=5681310", - "res=10", - "-p" - ], - "return_code": 0, - "run_time": 0.05014228820800781, - "stderr": [ - "" - ], - "stdout": "projection: 1 (UTM)\nzone: 32\ndatum: etrs89\nellipsoid: grs80\nnorth: 5681310\nsouth: 5628872\nwest: 366166\neast: 409135\nnsres: 9.99961861\newres: 9.99976728\nrows: 5244\ncols: 4297\ncells: 22533468\n" - } - ], - "process_results": {}, - "progress": { - "num_of_steps": 1, - "step": 1 - }, - "resource_id": "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", - "status": "finished", - "time_delta": 0.17178988456726074, - "timestamp": 1637920374.7826219, - "urls": { - "resources": [], - "status": "http://actinia-core-docker:8088/api/v1/resources/actinia-gdi/resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2" + "status": "http://localhost:8088/api/v3/resources/actinia-gdi/resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18" }, "user_id": "actinia-gdi" } }, - "actinia_core_url": "http://actinia-core-docker:8088/", - "actinia_gdi_batchid": 1, + "actinia_core_url": null, + "actinia_gdi_batchid": 5, "batch_description": { "jobs": [ { "list": [ { - "flags": "p", - "id": "g_region_test_0", + "id": "g_region_nonparallel_block1", "inputs": [ { - "param": "w", - "value": "366166" - }, - { - "param": "s", - "value": "5628872" - }, - { - "param": "e", - "value": "409135" - }, - { - "param": "n", - "value": "5681310" - }, - { - "param": "res", - "value": "10" + "param": "raster", + "value": "elevation@PERMANENT" } ], "module": "g.region" + }, + { + "id": "r_mapcalc_0_nonparallel_block1", + "inputs": [ + { + "param": "expression", + "value": "baum = elevation@PERMANENT * 2" + } + ], + "module": "r.mapcalc" } ], - "parallel": "true", + "parallel": "false", "version": "1" }, { "list": [ { - "id": "g_list_0", + "flags": "p", + "id": "g_region_1_parallel_block2", "inputs": [ { - "param": "mapset", - "value": "." - }, + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + }, + { + "id": "r_info_1_parallel_block2", + "inputs": [ { - "param": "type", - "value": "raster" + "param": "map", + "value": "elevation@PERMANENT" } ], - "module": "g.list" + "module": "r.info" } ], "parallel": "true", @@ -179,101 +95,102 @@ "list": [ { "flags": "p", - "id": "g_region_test_1", + "id": "g_region_2_parallel_block2", "inputs": [ { - "param": "w", - "value": "366166" - }, - { - "param": "s", - "value": "5628872" - }, - { - "param": "e", - "value": "409135" - }, - { - "param": "n", - "value": "5681310" - }, - { - "param": "res", - "value": "10" + "import_descr": { + "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", + "type": "raster" + }, + "param": "raster", + "value": "elev_ned_30m" } ], "module": "g.region" }, { - "id": "r_mapcalc_0", + "flags": "g", + "id": "r_univar_2_parallel_block2", "inputs": [ { - "param": "expression", - "value": "test_raster2 = 1" + "param": "map", + "value": "elev_ned_30m" } ], - "module": "r.mapcalc" - }, + "module": "r.univar", + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ { - "id": "r_info_0", + "flags": "p", + "id": "g_region_nonparallel_block3", "inputs": [ { - "param": "map", - "value": "test_raster2" + "param": "raster", + "value": "elevation@PERMANENT" } ], - "module": "r.info", - "stdout": { - "delimiter": ",", - "format": "table", - "id": "TEST" - } + "module": "g.region" } ], "parallel": "false", "version": "1" } - ], - "processing_host": "http://actinia-core-docker:8088/", - "processing_platform_name": "example_name" + ] }, - "status": "RUNNING", "creation_uuids": [ - "22638e4f-2e2e-4fd7-8221-cc5012993d2a", - "10153224-0945-4736-808e-90785f889686", - "7b31cb7d-f956-4f7a-8c16-c3eef34c72c9" + "768247dd-8dce-4aa2-bf96-51fb698e4fea", + "9163ca23-aad9-48c6-8eb4-80cc174d2090", + "5a91e046-faec-414d-8013-771747196126", + "ee72667f-248a-4200-8c2c-e6025fa80254" ], "idpk_jobs": [ - "652", - "650", - "651" + "17", + "18", + "19", + "20" ], - "process": "netdefinition", "jobs_status": [ + { + "actinia_core_jobid": "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", + "idpk_jobs": 17, + "status": "PENDING" + }, { "actinia_core_jobid": "None", - "idpk_jobs": 652, + "idpk_jobs": 18, "status": "PREPARING" }, { - "actinia_core_jobid": "resource_id-ed51442e-4fa2-43a0-af67-041d2f33f5b2", - "idpk_jobs": 650, - "status": "SUCCESS" + "actinia_core_jobid": "None", + "idpk_jobs": 19, + "status": "PREPARING" }, { - "actinia_core_jobid": "resource_id-7565f55f-b798-417e-8441-e148df6194d4", - "idpk_jobs": 651, - "status": "PENDING" + "actinia_core_jobid": "None", + "idpk_jobs": 20, + "status": "PREPARING" } ], + "process": "ephemeral", + "status": "PENDING", "summary": { "blocks": [ { "accepted": 1, "block_num": 1, "error": 0, - "finished": 1, - "parallel": 2, + "finished": 0, + "parallel": 1, "preparing": 0, "running": 0, "terminated": 0 @@ -283,6 +200,16 @@ "block_num": 2, "error": 0, "finished": 0, + "parallel": 2, + "preparing": 2, + "running": 0, + "terminated": 0 + }, + { + "accepted": 0, + "block_num": 3, + "error": 0, + "finished": 0, "parallel": 1, "preparing": 1, "running": 0, @@ -292,11 +219,11 @@ "status": { "accepted": 1, "error": 0, - "finished": 1, - "preparing": 1, + "finished": 0, + "preparing": 3, "running": 0, "terminated": 0 }, - "total": 3 + "total": 4 } } diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json new file mode 100644 index 0000000..1aa4887 --- /dev/null +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -0,0 +1,539 @@ +{ + "actinia_core_jobid": "resource_id-71264d56-4183-4d68-8544-5425254f5def", + "actinia_core_platform": null, + "actinia_core_platform_name": null, + "actinia_core_response": { + "accept_datetime": "2022-05-24 18:22:12.481060", + "accept_timestamp": 1653416532.4810588, + "api_info": { + "endpoint": "asyncparallelephermeralresource", + "method": "POST", + "path": "/api/v3/locations/nc_spm_08_grass7_root/processing_parallel", + "post_url": "http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel", + "request_url": "http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel" + }, + "datetime": "2022-05-24 18:22:12.990696", + "http_code": 200, + "message": "Processing successfully finished", + "process_chain_list": [ + { + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "outputs": [], + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "version": "1" + } + ], + "process_log": [ + { + "executable": "g.region", + "id": "g_region_2_parallel_block2", + "mapset_size": 407, + "parameter": [ + "raster=elev_ned_30m", + "-p" + ], + "return_code": 0, + "run_time": 0.1003119945526123, + "stderr": [ + "" + ], + "stdout": "projection: 99 (Lambert Conformal Conic)\nzone: 0\ndatum: nad83\nellipsoid: a=6378137 es=0.006694380022900787\nnorth: 228500\nsouth: 215000\nwest: 630000\neast: 645000\nnsres: 30\newres: 30\nrows: 450\ncols: 500\ncells: 225000\n" + }, + { + "executable": "r.univar", + "id": "r_univar_2_parallel_block2", + "mapset_size": 407, + "parameter": [ + "map=elev_ned_30m", + "-g" + ], + "return_code": 0, + "run_time": 0.10025954246520996, + "stderr": [ + "" + ], + "stdout": "n=225000\nnull_cells=0\ncells=225000\nmin=55.1736030578613\nmax=156.386520385742\nrange=101.212917327881\nmean=110.307571087138\nmean_of_abs=110.307571087138\nstddev=20.3119976726962\nvariance=412.577249455617\ncoeff_var=18.4139651272447\nsum=24819203.494606\n" + } + ], + "process_results": { + "stats": { + "cells": "225000", + "coeff_var": "18.4139651272447", + "max": "156.386520385742", + "mean": "110.307571087138", + "mean_of_abs": "110.307571087138", + "min": "55.1736030578613", + "n": "225000", + "null_cells": "0", + "range": "101.212917327881", + "stddev": "20.3119976726962", + "sum": "24819203.494606", + "variance": "412.577249455617" + } + }, + "progress": { + "num_of_steps": 2, + "step": 2 + }, + "resource_id": "resource_id-71264d56-4183-4d68-8544-5425254f5def", + "status": "finished", + "time_delta": 0.5096566677093506, + "timestamp": 1653416532.9906828, + "urls": { + "resources": [], + "status": "http://localhost:8088/api/v3/resources/actinia-gdi/resource_id-71264d56-4183-4d68-8544-5425254f5def" + }, + "user_id": "actinia-gdi" + }, + "actinia_core_url": null, + "batch_description": { + "jobs": [ + { + "list": [ + { + "id": "g_region_nonparallel_block1", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + }, + { + "id": "r_mapcalc_0_nonparallel_block1", + "inputs": [ + { + "param": "expression", + "value": "baum = elevation@PERMANENT * 2" + } + ], + "module": "r.mapcalc" + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_1_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + }, + { + "id": "r_info_1_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elevation@PERMANENT" + } + ], + "module": "r.info" + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "import_descr": { + "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", + "type": "raster" + }, + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region" + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_nonparallel_block3", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + } + ], + "parallel": "false", + "version": "1" + } + ] + }, + "batch_id": 3, + "creation_uuid": "49cdd55a-332b-4f66-ad40-c0b3e576f824", + "idpk_jobs": 11, + "job_description": { + "batch_description": { + "jobs": [ + { + "list": [ + { + "id": "g_region_nonparallel_block1", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "id": "r_mapcalc_0_nonparallel_block1", + "inputs": [ + { + "param": "expression", + "value": "baum = elevation@PERMANENT * 2" + } + ], + "module": "r.mapcalc", + "outputs": [] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_1_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "id": "r_info_1_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elevation@PERMANENT" + } + ], + "module": "r.info", + "outputs": [] + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "outputs": [], + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_nonparallel_block3", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region", + "outputs": [] + } + ], + "parallel": "false", + "version": "1" + } + ] + }, + "batch_id": 3, + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "outputs": [], + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "processing_block": 2, + "version": "1" + }, + "message": null, + "process": "ephemeral", + "processing_block": 2, + "rule_configuration": { + "batch_description": { + "jobs": [ + { + "list": [ + { + "id": "g_region_nonparallel_block1", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + }, + { + "id": "r_mapcalc_0_nonparallel_block1", + "inputs": [ + { + "param": "expression", + "value": "baum = elevation@PERMANENT * 2" + } + ], + "module": "r.mapcalc" + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_1_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + }, + { + "id": "r_info_1_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elevation@PERMANENT" + } + ], + "module": "r.info" + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "import_descr": { + "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", + "type": "raster" + }, + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region" + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "flags": "p", + "id": "g_region_nonparallel_block3", + "inputs": [ + { + "param": "raster", + "value": "elevation@PERMANENT" + } + ], + "module": "g.region" + } + ], + "parallel": "false", + "version": "1" + } + ] + }, + "batch_id": 3, + "list": [ + { + "flags": "p", + "id": "g_region_2_parallel_block2", + "inputs": [ + { + "param": "raster", + "value": "elev_ned_30m" + } + ], + "module": "g.region", + "outputs": [] + }, + { + "flags": "g", + "id": "r_univar_2_parallel_block2", + "inputs": [ + { + "param": "map", + "value": "elev_ned_30m" + } + ], + "module": "r.univar", + "outputs": [], + "stdout": { + "delimiter": "=", + "format": "kv", + "id": "stats" + } + } + ], + "parallel": "true", + "processing_block": 2, + "version": "1" + }, + "status": "SUCCESS", + "terraformer_id": null, + "terraformer_response": null, + "time_created": "Tue, 24 May 2022 18:22:09 GMT", + "time_ended": "Tue, 24 May 2022 18:22:12 GMT", + "time_estimated": null, + "time_started": null +} diff --git a/src/actinia_parallel_plugin/apidocs/helloworld.py b/src/actinia_parallel_plugin/apidocs/helloworld.py deleted file mode 100644 index 27d3b2a..0000000 --- a/src/actinia_parallel_plugin/apidocs/helloworld.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-present mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Hello World class -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - - -from actinia_parallel_plugin.model.response_models import ( - SimpleStatusCodeResponseModel, -) - - -describeHelloWorld_get_docs = { - # "summary" is taken from the description of the get method - "tags": ["example"], - "description": "Hello World example", - "responses": { - "200": { - "description": "This response returns the string 'Hello World!'", - "schema": SimpleStatusCodeResponseModel, - } - }, -} - -describeHelloWorld_post_docs = { - # "summary" is taken from the description of the get method - "tags": ["example"], - "description": "Hello World example with name", - "responses": { - "200": { - "description": "This response returns the string 'Hello World " - "NAME!'", - "schema": SimpleStatusCodeResponseModel, - }, - "400": { - "description": "This response returns a detail error message", - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string", - "description": "detailed message", - "example": "Missing name in JSON content", - } - }, - }, - }, - }, -} diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 79180ec..7375a62 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -24,145 +24,26 @@ __copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" +import os +import json from flask_restful_swagger_2 import Schema from actinia_parallel_plugin.apidocs.regeldatei import ( RegeldateiModel, - ProcessesProcOutputModel, ) +script_dir = os.path.dirname(os.path.abspath(__file__)) -class EnrichedProcInputBaseModel(Schema): - type = 'object' - properties = { - 'name': { - 'type': 'string', - 'description': 'Name of input data' - }, - 'type': { - 'type': 'string', - 'enum': ["GNOS", "DATABASE", "PARAMETER", "STATUS"], - 'description': 'Type of input. Can be "GNOS", "DATABASE", ' + - '"PARAMETER" or "STATUS"' - }, - 'geodata_meta': "string" # TODO: GeodataResponseModel - } - - -class EnrichedProcInputGnosModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - EnrichedProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - # }, - # ProcessesProcInputGnosModel, - # { - # '$ref': '#/definitions/ProcessesProcInputGnosModel' - } - - ] - - -class EnrichedProcInputDatabaseModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - EnrichedProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - # }, - # ProcessesProcInputDatabaseModel, - # { - # '$ref': '#/definitions/ProcessesProcInputDatabaseModel' - } - ] - - -class EnrichedProcInputParameterModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - EnrichedProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - # }, - # ProcessesProcInputParameterModel, - # { - # '$ref': '#/definitions/ProcessesProcInputParameterModel' - } - ] - - -class EnrichedProcInputStatusModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - EnrichedProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - # }, - # ProcessesProcInputStatusModel, - # { - # '$ref': '#/definitions/ProcessesProcInputStatusModel' - } - ] - - -class EnrichedProcInputModel(Schema): - """Request schema for creating a job""" - type = 'object' - # TODO: use oneOf (was not parsed in petstore) - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - EnrichedProcInputGnosModel, - EnrichedProcInputDatabaseModel, - EnrichedProcInputParameterModel, - EnrichedProcInputStatusModel, - { - '$ref': '#/definitions/EnrichedProcInputGnosModel' - }, - { - '$ref': '#/definitions/EnrichedProcInputDatabaseModel' - }, - { - '$ref': '#/definitions/EnrichedProcInputParameterModel' - }, - { - '$ref': '#/definitions/EnrichedProcInputStatusModel' - } - ] - - -class EnrichedProcModel(Schema): - """Request schema for creating a job""" - type = 'object' - properties = { - 'name': { - 'type': 'integer', - 'description': 'Name of process' - }, - 'input': { - 'type': 'array', - 'description': 'Definitions for process input data', - 'items': EnrichedProcInputModel - }, - 'output': { - 'type': 'array', - 'description': 'Definitions for process output data', - 'items': ProcessesProcOutputModel - }, - 'dependsOn': { - 'type': 'string', - 'description': 'List of names of processes on which this process' + - ' depends on. See also "status" as input parameter' - } - } +rel_path = "../apidocs/examples/jobs_get_docs_response_example.json" +abs_file_path = os.path.join(script_dir, rel_path) +print(abs_file_path) +with open(abs_file_path) as jsonfile: + jobs_get_docs_response_example = json.load(jsonfile) class EnrichedRegeldateiModel(Schema): """Request schema for creating a job""" + # TODO check if this is correct type = 'object' properties = { 'rule_area_id': { @@ -173,37 +54,32 @@ class EnrichedRegeldateiModel(Schema): 'type': 'string', 'description': 'Name of area where Regeldatei is valid' }, - 'feature_type': { - 'type': 'string', - 'description': 'Name of feature type to run job with' - }, 'feature_uuid': { 'type': 'string', 'description': 'Geonetwork UUID of feature type to run job with' }, - 'feature_source': EnrichedProcInputBaseModel, 'processing_platform': { 'type': 'string', - 'description': 'The actinia-core platform, either "openshift" or "vm". If platform is "vm" and no actinia_core_url is given, actinia-gdi will create a new VM.' + 'description': 'TODO' }, 'processing_platform_name': { 'type': 'string', - 'description': 'The actinia-core platform name. Only used to match a job to a VM if VM not started by actinia-gdi. Ideally it would contain the job type (actinia-core-pt or actinia-core-oc) and a unique ID.' + 'description': 'TODO (and a unique ID.)' }, 'processing_host': { 'type': 'string', - 'description': 'The actinia-core IP or URL in case the platform is not OpenShift and no new VM should be created by actinia-gdi' - }, - 'procs': { - 'type': 'array', - 'description': 'List of processes to run', - 'items': EnrichedProcModel - }, - 'geodata_meta': "string" # TODO: GeodataResponseModel + 'description': 'TODO (The actinia-core IP or URL)' + } + # 'procs': { + # 'type': 'array', + # 'description': 'List of processes to run', + # 'items': EnrichedProcModel + # } + # 'geodata_meta': "string" # TODO: GeodataResponseModel } # TODO add example # example = jobs_post_docs_request_example - required = ["feature_source", "procs"] + required = ["feature_source"] class ProcessesJobResponseModel(Schema): @@ -216,13 +92,9 @@ class ProcessesJobResponseModel(Schema): }, 'process': { 'type': 'string', - 'description': 'The process of the job, e.g standortsicherung ' + + 'description': 'The process of the job, e.g standortsicherung ' 'or potentialtrenches' }, - 'feature_type': { - 'type': 'string', - 'description': 'The feature type of the job' - }, 'rule_configuration': RegeldateiModel, 'job_description': EnrichedRegeldateiModel, 'time_created': { @@ -241,10 +113,6 @@ class ProcessesJobResponseModel(Schema): 'type': 'string', 'description': 'Timestamp when job was created' }, - 'metadata': { - 'type': 'string', - 'description': 'Not specified yet' - }, 'status': { 'type': 'string', 'description': 'Status of the Job', @@ -266,7 +134,8 @@ class ProcessesJobResponseModel(Schema): }, 'actinia_core_platform': { 'type': 'string', - 'description': 'The actinia-core platform, either "openshift" or "vm"' + 'description': 'The actinia-core platform, either "openshift" or ' + '"vm"' }, 'actinia_core_platform_name': { 'type': 'string', @@ -274,11 +143,14 @@ class ProcessesJobResponseModel(Schema): }, 'actinia_core_url': { 'type': 'string', - 'description': 'The actinia-core IP or URL where actinia-core is processing the job' + 'description': 'The actinia-core IP or URL where actinia-core is ' + 'processing the job' }, 'creation_uuid': { 'type': 'string', - 'description': 'A unique id for the job at creation time before idpk_jobs is known. (More unique than creation timestamp)' + 'description': 'A unique id for the job at creation time before ' + 'idpk_jobs is known. (More unique than creation ' + 'timestamp)' }, 'terraformer_id': { 'type': 'string', @@ -289,8 +161,7 @@ class ProcessesJobResponseModel(Schema): 'description': 'The Response/Status of terraformer' } } - # TODO add example - # example = jobs_post_docs_response_example + example = jobs_get_docs_response_example jobId_get_docs = { diff --git a/src/actinia_parallel_plugin/apidocs/regeldatei.py b/src/actinia_parallel_plugin/apidocs/regeldatei.py index 42c193e..8994224 100644 --- a/src/actinia_parallel_plugin/apidocs/regeldatei.py +++ b/src/actinia_parallel_plugin/apidocs/regeldatei.py @@ -29,9 +29,6 @@ from flask_restful_swagger_2 import Schema -from actinia_parallel_plugin.model.response_models import GeodataResponseModel - - # script_dir = os.path.dirname(os.path.abspath(__file__)) # null = "null" # @@ -58,7 +55,7 @@ class ProcessesProcInputBaseModel(Schema): 'type': { 'type': 'string', 'enum': ["GNOS", "DATABASE", "PARAMETER", "STATUS"], - 'description': 'Type of input. Can be "GNOS", "DATABASE", ' + + 'description': 'Type of input. Can be "GNOS", "DATABASE", ' '"PARAMETER" or "STATUS"' } } @@ -71,14 +68,14 @@ class ProcessesFilterModel(Schema): 'type': 'string', 'description': 'Not yet implemented or specified' }, - 'metadata_field': { - 'type': 'string', - 'description': 'Not yet implemented or specified' - }, - 'metadata_value': { - 'type': 'string', - 'description': 'Not yet implemented or specified' - }, + # 'metadata_field': { + # 'type': 'string', + # 'description': 'Not yet implemented or specified' + # }, + # 'metadata_value': { + # 'type': 'string', + # 'description': 'Not yet implemented or specified' + # }, 'operator': { 'type': 'string', 'description': 'Not yet implemented or specified' @@ -86,46 +83,6 @@ class ProcessesFilterModel(Schema): } -class ProcessesProcInputGnosModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - }, - { - 'type': 'object', - 'properties': { - 'tags': { - 'type': 'array', - 'description': 'Geonetwork tags to filter data', - 'items': { - 'type': 'string' - } - }, - 'uuid': { - 'type': 'string', - 'description': 'Geonetwork UUID to identify certain record' - }, - 'attributes': { - 'type': 'array', - 'description': 'Database attribute of data source to use', - 'items': { - 'type': 'string' - } - }, - 'filter': { - 'type': 'array', - 'description': 'Not yet implemented or specified', - 'items': ProcessesFilterModel - } - } - } - - ] - - class ProcessesProcInputDatabaseModel(Schema): """Request schema for creating a job""" allOf = [ @@ -171,7 +128,7 @@ class ProcessesProcInputParameterModel(Schema): 'properties': { 'value': { 'type': 'array', - 'description': 'Array of input parameter. Can be int, ' + + 'description': 'Array of input parameter. Can be int, ' 'float or string', 'items': { # TODO: find out how to allow multiple types @@ -200,7 +157,7 @@ class ProcessesProcInputFileModel(Schema): 'properties': { 'value': { 'type': 'array', - 'description': 'Array of input parameter. Can be int, ' + + 'description': 'Array of input parameter. Can be int, ' 'float or string', 'items': { # TODO: find out how to allow multiple types @@ -225,7 +182,7 @@ class ProcessesProcInputStatusModel(Schema): 'properties': { 'status': { 'type': 'string', - 'description': 'Status of another process as input. See ' + + 'description': 'Status of another process as input. See ' 'also "dependsOn"' } } @@ -239,14 +196,10 @@ class ProcessesProcInputModel(Schema): # TODO: use oneOf (was not parsed in petstore) allOf = [ # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputGnosModel, ProcessesProcInputDatabaseModel, ProcessesProcInputParameterModel, ProcessesProcInputFileModel, ProcessesProcInputStatusModel, - { - '$ref': '#/definitions/ProcessesProcInputGnosModel' - }, { '$ref': '#/definitions/ProcessesProcInputDatabaseModel' }, @@ -305,7 +258,7 @@ class ProcessesProcModel(Schema): }, 'dependsOn': { 'type': 'string', - 'description': 'List of names of processes on which this process' + + 'description': 'List of names of processes on which this process' ' depends on. See also "status" as input parameter' } } @@ -313,6 +266,7 @@ class ProcessesProcModel(Schema): class RegeldateiModel(Schema): """Request schema to create a job""" + # TODO check if this is correct type = 'object' properties = { 'rule_area_id': { @@ -323,10 +277,6 @@ class RegeldateiModel(Schema): 'type': 'string', 'description': 'Name of area where Regeldatei is valid' }, - 'feature_type': { - 'type': 'string', - 'description': 'Name of feature type to run job with' - }, 'feature_uuid': { 'type': 'string', 'description': 'Geonetwork UUID of feature type to run job with' @@ -334,30 +284,20 @@ class RegeldateiModel(Schema): 'feature_source': ProcessesProcInputModel, 'processing_platform': { 'type': 'string', - 'description': 'The actinia-core platform, either "openshift" or "vm". If platform is "vm" and no actinia_core_url is given, actinia-gdi will create a new VM.' + 'description': 'TODO' }, 'processing_platform_name': { 'type': 'string', - 'description': 'The actinia-core platform name. Only used to match a job to a VM if VM not started by actinia-gdi. Ideally it would contain the job type (actinia-core-pt or actinia-core-oc) and a unique ID.' + 'description': 'TODO (and a unique ID.)' }, 'processing_host': { 'type': 'string', - 'description': 'The actinia-core IP or URL in case the platform is not OpenShift and no new VM should be created by actinia-gdi' + 'description': 'TODO (The actinia-core IP or URL)' }, 'procs': { 'type': 'array', 'description': 'List of processes to run', 'items': ProcessesProcModel - }, - 'geodata_meta': GeodataResponseModel + } } - # examples = { - # 'zero': { - # 'value': sos_jobs_post_docs_request_example - # }, - # 'max': { - # 'value': pt_jobs_post_docs_request_example - # } - # } - # example = sos_jobs_post_docs_request_example required = ["feature_source", "procs"] diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index b154ea9..7d64945 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -104,7 +104,7 @@ def checkBatchProcessChain(jsonDict): """ bpc = BatchProcessChain(**jsonDict) - bpc.feature_type = "default" + # bpc.feature_type = "default" # check consistency bpc_dict = bpc.to_struct() if len(bpc_dict["jobs"]) == 0: @@ -140,7 +140,7 @@ def createBatch(jsonDict, process, batchid): # assign the model process_chain = SingleJob(**job) # might be needed (?) - process_chain.feature_type = "null" + # process_chain.feature_type = "null" job_in_db = insertJob(job, process, process_chain) jobs_in_db.append(job_in_db) return jobs_in_db @@ -267,7 +267,8 @@ def getAllBatchIds(): # def getAllBatches(process): -# """ Function to return all jobs that are part of a batch from the database +# """ Function to return all jobs that are part of a batch from the +# database # """ # result_list = [] # batchids = getAllBatchIds() @@ -304,13 +305,13 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, process_chain["list"] = job["rule_configuration"]["list"] process_chain["version"] = job["rule_configuration"]["version"] jobid = job["idpk_jobs"] - start_kwargs = { - "process": job["process"], - "process_chain": process_chain, - "jobid": job["idpk_jobs"], - # "actinia_core_platform": job["actinia_core_platform"], - # "actinia_core_url": job["actinia_core_url"] - } + # start_kwargs = { + # "process": job["process"], + # "process_chain": process_chain, + # "jobid": job["idpk_jobs"], + # # "actinia_core_platform": job["actinia_core_platform"], + # # "actinia_core_url": job["actinia_core_url"] + # } mapset_name_parallel = mapset_name if mapset_suffix != "" and mapset_name is not None: mapset_name_parallel += f"{mapset_suffix}{num}" diff --git a/src/actinia_parallel_plugin/core/ephemeral_processing.py b/src/actinia_parallel_plugin/core/ephemeral_processing.py index 57add45..954fbf7 100644 --- a/src/actinia_parallel_plugin/core/ephemeral_processing.py +++ b/src/actinia_parallel_plugin/core/ephemeral_processing.py @@ -84,7 +84,7 @@ def _update_and_check_batch_jobs(self): jobs_from_batch, block) if block_done is True and block < max(all_blocks): next_block = block + 1 - next_jobs = startProcessingBlock( + startProcessingBlock( jobs_from_batch, next_block, self.batch_id, diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index eb7cc11..ceb5677 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -26,7 +26,6 @@ from actinia_parallel_plugin.core.jobtable import ( getJobById, - # getJobByResource, insertNewJob, updateJobByID, ) @@ -70,7 +69,8 @@ def insertJob(jsonDict, process, process_chain): # if (regeldatei.processing_host): # actinia_core_url = regeldatei.processing_host # if not actinia_core_url.startswith('http'): - # actinia_core_url = ACTINIACORE_VM.scheme + '://' + actinia_core_url + # actinia_core_url = ACTINIACORE_VM.scheme + '://' + \ + # actinia_core_url # if len(actinia_core_url.split(':')) == 2: # actinia_core_url += ':' + ACTINIACORE_VM.port # @@ -81,7 +81,6 @@ def insertJob(jsonDict, process, process_chain): jsonDict, process_chain_struct, process, - process_chain.feature_type, # actinia_core_url, # actinia_core_platform, # actinia_core_platform_name diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index e7aeed5..a99bdc4 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -29,6 +29,7 @@ from playhouse.shortcuts import model_to_dict from peewee import Expression, AutoField, OperationalError +from time import sleep from uuid import uuid4 from yoyo import read_migrations from yoyo import get_backend @@ -54,7 +55,8 @@ def applyMigrations(): 'postgres://%s:%s@%s/%s?schema=%s' % (JOBTABLE.user, JOBTABLE.pw, JOBTABLE.host, JOBTABLE.database, JOBTABLE.schema)) - migrations = read_migrations('actinia_parallel_plugin/resources/migrations') + migrations = read_migrations( + 'actinia_parallel_plugin/resources/migrations') with backend.lock(): backend.apply_migrations(backend.to_apply(migrations)) @@ -233,7 +235,7 @@ def insertNewJob( rule_configuration, job_description, process, - feature_type, + # feature_type, actinia_core_url=None, actinia_core_platform=None, actinia_core_platform_name=None @@ -243,7 +245,6 @@ def insertNewJob( Args: rule_configuration (dict): original regeldatei job_description (TODO): enriched regeldatei with geometadata - feature_type (string): feature_type name actinia_core_url (string): url where processing will run actinia_core_platform (string): platform where processing will run @@ -261,7 +262,6 @@ def insertNewJob( 'process': process, 'status': 'PREPARING', 'time_created': utcnow, - 'feature_type': feature_type, 'actinia_core_url': actinia_core_url, 'actinia_core_platform': actinia_core_platform, 'actinia_core_platform_name': actinia_core_platform_name, @@ -326,7 +326,10 @@ def updateJobByID( status = 'TERMINATED' utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') - record, err = getJobById(jobid) + record = None + while record is None: + record, err = getJobById(jobid) + sleep(1) dbStatus = record['status'] try: diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index 24e3b22..a80f32e 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -24,28 +24,17 @@ __copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" -import json import pickle -# from actinia_core.rest.base.resource_base import ResourceBase from actinia_core.core.common.redis_interface import enqueue_job -# from actinia_parallel_plugin.core.batches import ( -# # createBatch, -# # createBatchId, -# # createBatchResponseDict, -# # getJobsByBatchId, -# ) from actinia_parallel_plugin.core.jobtable import ( getJobById, ) -# from actinia_parallel_plugin.model.response_models import ( -# SimpleStatusCodeResponseModel, -# ) from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log -# from actinia_parallel_plugin.core.persistent_processing import start_job -from actinia_parallel_plugin.core.parallel_resource_base import ParallelResourceBase +from actinia_parallel_plugin.core.parallel_resource_base import \ + ParallelResourceBase class AsyncParallelJobResource(ParallelResourceBase): @@ -87,14 +76,20 @@ def start_parallel_job(self, process, block): mapset_name=self.mapset_name ) if rdc: - - if process == "persistent": - from actinia_parallel_plugin.core.persistent_processing import \ + if process == "ephemeral": + from actinia_parallel_plugin.core.ephemeral_processing import \ start_job - # # for debugging (works not so gogd with parallel processing) - # from actinia_parallel_plugin.core.persistent_processing import \ - # ParallelPersistentProcessing - # processing = ParallelPersistentProcessing( + # # for debugging comment enqueue_job(...) and use the + # # following commented lines + # for var in [ + # 'GISRC', 'GISBASE', 'LD_LIBRARY_PATH', + # 'GRASS_ADDON_PATH', 'GIS_LOCK']: + # import os + # if var in os.environ: + # del os.environ[var] + # from actinia_parallel_plugin.core.ephemeral_processing \ + # import ParallelEphemeralProcessing + # processing = ParallelEphemeralProcessing( # rdc, self.batch_id, block, self.job_id, # self.user, # self.request_url, @@ -104,47 +99,46 @@ def start_parallel_job(self, process, block): # self.path, # self.base_status_url) # processing.run() - elif process == "ephemeral": - from actinia_parallel_plugin.core.ephemeral_processing import \ - start_job - # for debugging - for var in [ - 'GISRC', 'GISBASE', 'LD_LIBRARY_PATH', - 'GRASS_ADDON_PATH', 'GIS_LOCK']: - import os - if var in os.environ: - del os.environ[var] - from actinia_parallel_plugin.core.ephemeral_processing import \ - ParallelEphemeralProcessing - processing = ParallelEphemeralProcessing( - rdc, self.batch_id, block, self.job_id, - self.user, - self.request_url, - self.post_url, - self.endpoint, - self.method, - self.path, - self.base_status_url) - processing.run() + # elif process == "persistent": + # from actinia_parallel_plugin.core.persistent_processing \ + # import start_job + # # TODO + # # # for debugging + # # from actinia_parallel_plugin.core.persistent_processing \ + # # import ParallelPersistentProcessing + # # processing = ParallelPersistentProcessing( + # # rdc, self.batch_id, block, self.job_id, + # # self.user, + # # self.request_url, + # # self.post_url, + # # self.endpoint, + # # self.method, + # # self.path, + # # self.base_status_url) + # # processing.run() else: - # TODO change start_job import - from actinia_parallel_plugin.core.persistent_processing import \ - start_job - # enqueue_job( - # self.job_timeout, - # start_job, - # rdc, - # self.batch_id, - # block, - # self.job_id, - # self.user, - # self.request_url, - # self.post_url, - # self.endpoint, - # self.method, - # self.path, - # self.base_status_url - # ) + msg = f"Process '{process}' not yet supported!" + log.error(msg) + _, response_model = pickle.loads(self.response_data) + response_model["status"] = "error" + response_model["message"] = msg + job = updateJob(self.resource_id, response_model, self.job_id) + return job + enqueue_job( + self.job_timeout, + start_job, + rdc, + self.batch_id, + block, + self.job_id, + self.user, + self.request_url, + self.post_url, + self.endpoint, + self.method, + self.path, + self.base_status_url + ) # update job in jobtable self.response_data = self.resource_logger.get( @@ -156,25 +150,3 @@ def start_parallel_job(self, process, block): def get_job_entry(self): """Return job entry by requesting jobtable from db.""" return getJobById(self.job_id)[0] - - # # first_jobs = self._start_processing_block(jobs_in_db, 1) - # def _start_processing_block(self, jobs, block): - # """Starts first processing block of jobs from batch process. - # """ - # jobs_to_start = [ - # job for job in jobs if job["processing_block"] == block] - # jobs_responses = [] - # for job in jobs_to_start: - # process_chain = dict() - # process_chain["list"] = job["rule_configuration"]["list"] - # process_chain["version"] = job["rule_configuration"]["version"] - # start_kwargs = { - # "process": job["process"], - # "process_chain": process_chain, - # "jobid": job["idpk_jobs"], - # # "actinia_core_platform": job["actinia_core_platform"], - # # "actinia_core_url": job["actinia_core_url"] - # } - # job_entry = self._start_job(**start_kwargs) - # jobs_responses.append(job_entry) - # return jobs_responses diff --git a/src/actinia_parallel_plugin/core/parallel_resource_base.py b/src/actinia_parallel_plugin/core/parallel_resource_base.py index cddcbe2..8e7be68 100644 --- a/src/actinia_parallel_plugin/core/parallel_resource_base.py +++ b/src/actinia_parallel_plugin/core/parallel_resource_base.py @@ -30,14 +30,10 @@ from flask_restful_swagger_2 import Resource -# from actinia_core.core.common.app import flask_api -# from actinia_core.rest.resource_streamer import RequestStreamerResource - from actinia_core.core.common.config import global_config from actinia_core.core.messages_logger import MessageLogger from actinia_core.core.resources_logger import ResourceLogger from actinia_core.rest.base.resource_base import ResourceBase -# from actinia_core.rest.resource_management import ResourceManager from actinia_core.models.response_models import ( create_response_from_model, ApiInfoModel, @@ -105,7 +101,8 @@ def __init__(self, user, request_url, endpoint, method, path, # Generate the status URL self.status_url = f"{base_status_url}{self.resource_id}" - if global_config.FORCE_HTTPS_URLS is True and "http://" in self.status_url: + if (global_config.FORCE_HTTPS_URLS is True + and "http://" in self.status_url): self.status_url = self.status_url.replace("http://", "https://") self.request_url = request_url @@ -139,8 +136,10 @@ def preprocess(self, has_json=True, has_xml=False, - Send an accept entry to the resource redis database Args: - has_json (bool):Set True if the request has JSON data, False otherwise - has_xml (bool):Set True if the request has XML data, False otherwise + has_json (bool): Set True if the request has JSON data, False + otherwise + has_xml (bool): Set True if the request has XML data, False + otherwise location_name (str): The name of the location to work in mapset_name (str): The name of the target mapset in which the computation should be performed @@ -153,26 +152,17 @@ def preprocess(self, has_json=True, has_xml=False, the self.response_data variable to send a response. """ - # if has_json is True and has_xml is True: - # if request.is_json is True: - # self.request_data = request.get_json() - # else: - # if self.check_for_xml() is False: - # return None - # elif has_xml is True: - # if self.check_for_xml() is False: - # return None - # elif has_json is True: - # if self.check_for_json() is False: - # return None # Compute the job timeout of the worker queue from the user credentials - process_time_limit = self.user_credentials["permissions"]["process_time_limit"] - process_num_limit = self.user_credentials["permissions"]["process_num_limit"] + process_time_limit = self.user_credentials["permissions"][ + "process_time_limit"] + process_num_limit = self.user_credentials["permissions"][ + "process_num_limit"] self.job_timeout = int(process_time_limit * process_num_limit * 20) # Create the resource URL base and use a placeholder for the file name - # The placeholder __None__ must be replaced by the resource URL generator + # The placeholder __None__ must be replaced by the resource URL + # generator # TODO check if this works self.resource_url_base = f"{self.status_url}/__None__" @@ -203,21 +193,23 @@ def preprocess(self, has_json=True, has_xml=False, # Return the ResourceDataContainer that includes all # required data for the asynchronous processing - return ResourceDataContainer(grass_data_base=self.grass_data_base, - grass_user_data_base=self.grass_user_data_base, - grass_base_dir=self.grass_base_dir, - request_data=self.request_data, - user_id=self.user_id, - user_group=self.user_group, - user_credentials=self.user_credentials, - resource_id=self.resource_id, - iteration=self.iteration, - status_url=self.status_url, - api_info=self.api_info, - resource_url_base=self.resource_url_base, - orig_time=self.orig_time, - orig_datetime=self.orig_datetime, - config=global_config, - location_name=location_name, - mapset_name=mapset_name, - map_name=map_name) + return ResourceDataContainer( + grass_data_base=self.grass_data_base, + grass_user_data_base=self.grass_user_data_base, + grass_base_dir=self.grass_base_dir, + request_data=self.request_data, + user_id=self.user_id, + user_group=self.user_group, + user_credentials=self.user_credentials, + resource_id=self.resource_id, + iteration=self.iteration, + status_url=self.status_url, + api_info=self.api_info, + resource_url_base=self.resource_url_base, + orig_time=self.orig_time, + orig_datetime=self.orig_datetime, + config=global_config, + location_name=location_name, + mapset_name=mapset_name, + map_name=map_name + ) diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py index 5ecd90f..40ca868 100644 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ b/src/actinia_parallel_plugin/core/persistent_processing.py @@ -75,7 +75,8 @@ def _execute(self, skip_permission_check=False): - Create the temporal database - Initialize the GRASS environment and create the temporary mapset - Run the modules - - Parse the stdout output of the modules and generate the module results + - Parse the stdout output of the modules and generate the module + results Args: skip_permission_check (bool): If set True, the permission checks of @@ -91,11 +92,9 @@ def _execute(self, skip_permission_check=False): if self.rdc.iteration is not None: process_list = \ self._create_temporary_grass_environment_and_process_list_for_iteration( - # process_chain=process_chain, skip_permission_check=skip_permission_check) else: process_list = self._create_temporary_grass_environment_and_process_list( - # process_chain=process_chain, skip_permission_check=skip_permission_check) # Run all executables @@ -103,7 +102,6 @@ def _execute(self, skip_permission_check=False): # Parse the module sdtout outputs and create the results self._parse_module_outputs() - # def run(self, process_chain): def run(self): """This function will run the processing and will catch and process any Exceptions that were raised while processing. Call this function @@ -121,7 +119,6 @@ def run(self): """ try: # Run the _execute function that does all the work - # self._execute(process_chain=process_chain) self._execute() except AsyncProcessTermination as e: self.run_state = {"terminated": str(e)} @@ -192,7 +189,7 @@ def _update_and_check_batch_jobs(self): resource_id = self.resource_id response_data = self.resource_logger.get( self.user_id, self.resource_id) - http_code, response_model = pickle.loads(response_data) + _, response_model = pickle.loads(response_data) updateJob(resource_id, response_model, self.jobid) if "finished" == response_model["status"]: @@ -207,7 +204,7 @@ def _update_and_check_batch_jobs(self): jobs_from_batch, block) if block_done is True and block < max(all_blocks): next_block = block + 1 - next_jobs = startProcessingBlock( + startProcessingBlock( jobs_from_batch, next_block, self.batch_id, @@ -233,8 +230,3 @@ def _update_and_check_batch_jobs(self): def start_job(*args): processing = ParallelPersistentProcessing(*args) processing.run() - -# def start_job(*args): -# process_chain = json.loads(args[-1]) -# processing = ParallelPersistentProcessing(*args[:-1]) -# processing.run(process_chain) diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index f65933f..1537da1 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -27,8 +27,8 @@ from actinia_parallel_plugin.api.batch import BatchJobsId from actinia_parallel_plugin.api.job import JobId -from actinia_parallel_plugin.api.parallel_processing import \ - AsyncParallelPersistentResource +# from actinia_parallel_plugin.api.parallel_processing import \ +# AsyncParallelPersistentResource from actinia_parallel_plugin.api.parallel_ephemeral_processing import \ AsyncParallelEphermeralResource from actinia_parallel_plugin.core.jobtable import initJobDB, applyMigrations @@ -39,19 +39,16 @@ def create_endpoints(flask_api): apidoc = flask_api - # POST parallel ephemeral processing apidoc.add_resource( AsyncParallelEphermeralResource, "/locations//processing_parallel") - - # POST parallel persistent processing - apidoc.add_resource( - AsyncParallelPersistentResource, - "/locations//mapsets/" - "/processing_parallel") - # apidoc.add_resource(ParallelProcessingResource, "/processing_parallel") + # # POST parallel persistent processing + # apidoc.add_resource( + # AsyncParallelPersistentResource, + # "/locations//mapsets/" + # "/processing_parallel") # GET batch jobs by ID apidoc.add_resource( diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 3b3829f..2ea8e02 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -25,23 +25,18 @@ __maintainer__ = "mundialis GmbH % Co. KG" import os -# import json +import json from flask_restful_swagger_2 import Schema from actinia_core.models.process_chain import ProcessChainModel script_dir = os.path.dirname(os.path.abspath(__file__)) -# TODO -# rel_path = ("../apidocs/examples/" -# + "batchjob_post_response_example.json") -# print(script_dir) -# print(rel_path) -# abs_file_path = os.path.join(script_dir, rel_path) -# print(abs_file_path) -# print(os.path.isdir(abs_file_path)) -# with open(abs_file_path) as jsonfile: -# batchjob_post_docs_response_example = json.load(jsonfile) +rel_path = "../apidocs/examples/batchjob_post_response_example.json" +abs_file_path = os.path.join(script_dir, rel_path) +print(abs_file_path) +with open(abs_file_path) as jsonfile: + batchjob_post_docs_response_example = json.load(jsonfile) class BatchJobsSummaryModel(Schema): @@ -166,12 +161,15 @@ class BatchProcessChainModel(Schema): 'type (actinia-core-pt or actinia-core-oc) and ' 'a unique ID.' }, - 'jobs': {'type': 'array', - 'items': ProcessChainModel, - 'description': "A list of process chains (jobs) that should " - "be executed in parallel or sequentially " - "in the order provided by the list."} + 'jobs': { + 'type': 'array', + 'items': ProcessChainModel, + 'description': "A list of process chains (jobs) that should " + "be executed in parallel or sequentially " + "in the order provided by the list." } + } + required = ["jobs"] class BatchJobResponseModel(Schema): @@ -264,7 +262,6 @@ class BatchJobResponseModel(Schema): } } }, - 'summary': BatchJobsSummaryModel } - # example = batchjob_post_docs_response_example + example = batchjob_post_docs_response_example diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index ebd690e..8edbf89 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -63,14 +63,14 @@ class Job(BaseModel): """ idpk_jobs = AutoField() process = CharField(null=True) - feature_type = CharField(null=True) + # feature_type = CharField(null=True) rule_configuration = BinaryJSONField(null=True) job_description = BinaryJSONField(null=True) time_created = DateTimeField(null=True) time_started = DateTimeField(null=True) time_estimated = DateTimeField(null=True) time_ended = DateTimeField(null=True) - metadata = CharField(null=True) + # metadata = CharField(null=True) status = CharField(null=True) actinia_core_response = BinaryJSONField(null=True) actinia_core_jobid = CharField(null=True) diff --git a/src/actinia_parallel_plugin/resources/config.py b/src/actinia_parallel_plugin/resources/config.py index bc77804..f098d98 100644 --- a/src/actinia_parallel_plugin/resources/config.py +++ b/src/actinia_parallel_plugin/resources/config.py @@ -26,7 +26,7 @@ import configparser import os -from pathlib import Path +# from pathlib import Path # # config can be overwritten by mounting *.ini files into folders inside diff --git a/src/actinia_parallel_plugin/resources/logging.py b/src/actinia_parallel_plugin/resources/logging.py index f5f3e33..45e0a4f 100644 --- a/src/actinia_parallel_plugin/resources/logging.py +++ b/src/actinia_parallel_plugin/resources/logging.py @@ -45,9 +45,11 @@ def setLogFormat(veto=None): logformat = "" if LOGCONFIG.type == 'json' and not veto: - logformat = CustomJsonFormatter('%(time) %(level) %(component) %(module)' - '%(message) %(pathname) %(lineno)' - '%(processName) %(threadName)') + logformat = CustomJsonFormatter( + '%(time) %(level) %(component) %(module)' + '%(message) %(pathname) %(lineno)' + '%(processName) %(threadName)' + ) else: logformat = ColoredFormatter( '%(log_color)s[%(asctime)s] %(levelname)-10s: %(name)s.%(module)-' diff --git a/test_postbodies/parallel_ephemeral_processing.json b/test_postbodies/parallel_ephemeral_processing.json index 3b08057..f07d2c8 100644 --- a/test_postbodies/parallel_ephemeral_processing.json +++ b/test_postbodies/parallel_ephemeral_processing.json @@ -47,7 +47,15 @@ "module": "g.region", "id": "g_region_2_parallel_block2", "inputs":[ - {"param": "raster", "value": "elevation@PERMANENT"} + { + "import_descr": + { + "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", + "type": "raster" + }, + "param": "raster", + "value": "elev_ned_30m" + } ], "flags": "p" }, @@ -55,8 +63,10 @@ "module": "r.univar", "id": "r_univar_2_parallel_block2", "inputs":[ - {"param": "map", "value": "elevation@PERMANENT"} - ] + {"param": "map", "value": "elev_ned_30m"} + ], + "stdout": {"id": "stats", "format": "kv", "delimiter": "="}, + "flags": "g" } ], "parallel": "true", From 794fa3e832dcc6d18790f42dcdc3f745dccf9f11 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 25 May 2022 08:04:20 +0200 Subject: [PATCH 09/47] start with tests --- .../actinia-parallel-plugin-test/Dockerfile | 2 +- tests/integrationtests/test_helloworld.py | 83 ---------- .../test_parallel_ephemeral_processing.py | 153 ++++++++++++++++++ tests/unittests/test_transformation.py | 39 ----- tests_with_redis.sh | 2 + 5 files changed, 156 insertions(+), 123 deletions(-) delete mode 100644 tests/integrationtests/test_helloworld.py create mode 100644 tests/integrationtests/test_parallel_ephemeral_processing.py delete mode 100644 tests/unittests/test_transformation.py diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index a821e7d..c80ce73 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -9,7 +9,7 @@ ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia-parallel-plugin-test ENV DEFAULT_CONFIG_PATH /etc/default/actinia-parallel-plugin-test # install things only for tests -RUN apk add redis +RUN apk add redis postgresql RUN pip3 install iniconfig colorlog pwgen # COPY docker/actinia-parallel-plugin-test/start.sh /src/start.sh diff --git a/tests/integrationtests/test_helloworld.py b/tests/integrationtests/test_helloworld.py deleted file mode 100644 index 474c85c..0000000 --- a/tests/integrationtests/test_helloworld.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-present mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Hello World test -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - - -import json -import pytest -from flask import Response - -from ..test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX - - -class ActiniaParallelProcessingTest(ActiniaResourceTestCaseBase): - @pytest.mark.integrationtest - def test_get_processing_parallel(self): - """Test the get method of the /processing_parallel endpoint""" - resp = self.server.get(URL_PREFIX + "/processing_parallel") - - assert type(resp) is Response, "The response is not of type Response" - assert resp.status_code == 200, "The status code is not 200" - assert hasattr(resp, "json"), "The response has no attribute 'json'" - assert "message" in resp.json, ( - "There is no 'message' inside the " "response" - ) - assert resp.json["message"] == "Hello world!", ( - "The response message" " is wrong" - ) - - @pytest.mark.integrationtest - def test_post_processing_parallel(self): - """Test the post method of the /processing_parallel endpoint""" - postbody = {"name": "test"} - resp = self.server.post( - URL_PREFIX + "/processing_parallel", - headers=self.user_auth_header, - data=json.dumps(postbody), - content_type="application/json", - ) - assert type(resp) is Response, "The response is not of type Response" - assert resp.status_code == 200, "The status code is not 200" - assert hasattr(resp, "json"), "The response has no attribute 'json'" - assert "message" in resp.json, ( - "There is no 'message' inside the " "response" - ) - assert resp.json["message"] == "Hello world TEST!", ( - "The response " "message is wrong" - ) - - @pytest.mark.integrationtest - def test_post_processing_parallel_error(self): - """Test the post method of the /processing_parallel endpoint""" - postbody = {"namee": "test"} - resp = self.server.post( - URL_PREFIX + "/processing_parallel", - headers=self.user_auth_header, - data=json.dumps(postbody), - content_type="application/json", - ) - assert type(resp) is Response, "The response is not of type Response" - assert resp.status_code == 400, "The status code is not 400" - assert resp.data == b"Missing name in JSON content" diff --git a/tests/integrationtests/test_parallel_ephemeral_processing.py b/tests/integrationtests/test_parallel_ephemeral_processing.py new file mode 100644 index 0000000..940971c --- /dev/null +++ b/tests/integrationtests/test_parallel_ephemeral_processing.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel ephemeral processing tests +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + + +import json +import pytest +from flask import Response + +from ..test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX + +PC = """{ + "jobs": [ + { + "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block1", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ] + }, + { + "module": "r.mapcalc", + "id": "r_mapcalc_0_nonparallel_block1", + "inputs":[ + {"param": "expression", "value": "baum = elevation@PERMANENT * 2"} + ] + } + ], + "parallel": "false", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_1_parallel_block2", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "flags": "p" + }, + { + "module": "r.info", + "id": "r_info_1_parallel_block2", + "inputs":[ + {"param": "map", "value": "elevation@PERMANENT"} + ] + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_2_parallel_block2", + "inputs":[ + { + "import_descr": + { + "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", + "type": "raster" + }, + "param": "raster", + "value": "elev_ned_30m" + } + ], + "flags": "p" + }, + { + "module": "r.univar", + "id": "r_univar_2_parallel_block2", + "inputs":[ + {"param": "map", "value": "elev_ned_30m"} + ], + "stdout": {"id": "stats", "format": "kv", "delimiter": "="}, + "flags": "g" + } + ], + "parallel": "true", + "version": "1" + }, + { + "list": [ + { + "module": "g.region", + "id": "g_region_nonparallel_block3", + "inputs":[ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "flags": "p" + } + ], + "parallel": "false", + "version": "1" + } + ] +} +""" + + +class ActiniaParallelProcessingTest(ActiniaResourceTestCaseBase): + + location = "nc_spm_08" + base_url = f"{URL_PREFIX}/locations/{location}" + content_type = "application/json" + + @pytest.mark.integrationtest + def test_post_parallel_ephemeral_processing(self): + """Test the post method of the parallel ephemeral processing endpoint + """ + url = f"{self.base_url}/processing_parallel" + + rv = self.server.post( + url, + headers=self.user_auth_header, + content_type=self.content_type, + data=PC, + ) + import pdb; pdb.set_trace() + resp = self.waitAsyncStatusAssertHTTP( + rv, + headers=self.user_auth_header, + http_status=200, + status="finished", + ) + assert "process_results" in resp, "No 'process_results' in response" + assert resp["process_results"] == ["grid1", "grid2", "grid3", "grid4"] diff --git a/tests/unittests/test_transformation.py b/tests/unittests/test_transformation.py deleted file mode 100644 index 0b742b0..0000000 --- a/tests/unittests/test_transformation.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-present mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -First test -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - -import pytest -from actinia_parallel_plugin.core.example import transform_input - - -@pytest.mark.unittest -@pytest.mark.parametrize( - "inp,ref_out", - [("test", "Hello world TEST!"), ("bla23", "Hello world BLA23!")], -) -def test_transform_input(inp, ref_out): - """Test for tranform_input function.""" - out = transform_input(inp) - assert out == ref_out, f"Wrong result from transform_input for {inp}" diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 57405bf..323860d 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -5,6 +5,8 @@ redis-server & sleep 1 redis-cli ping +# TODO start postgresql + # start webhook server webhook-server --host "0.0.0.0" --port "5005" & sleep 10 From 1afab11daac9ecb2882d9987029533a9155efa8b Mon Sep 17 00:00:00 2001 From: Anika Weinmann <37300249+anikaweinmann@users.noreply.github.com> Date: Wed, 1 Jun 2022 08:16:06 +0200 Subject: [PATCH 10/47] Apply suggestions from code review Co-authored-by: Markus Neteler --- README.md | 6 +++--- setup.cfg | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 732b310..927b317 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ gunicorn -b 0.0.0.0:8088 -w 1 --access-logfile=- -k gthread actinia_core.main:fl reset && (cd /src/actinia-parallel-plugin && python3 setup.py install) && gunicorn -b 0.0.0.0:8088 -w 3 --access-logfile=- -k gthread actinia_core.main:flask_app ``` -### Postgis -Connect to postgis DB from actinia-core docker container: +### PostGIS +Connect to PostGIS DB from actinia-core docker container: ``` psql -U actinia -h postgis -d gis ``` @@ -112,7 +112,7 @@ You can also start a **persistent** batch job via: curl -u actinia-gdi:actinia-gdi -X POST -H 'Content-Type: application/json' -d @test_postbodies/parallel_processing.json http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/mapsets/test_mapset/processing_parallel | jq ``` Hereby, the parallel started process chains will be computed in mapsets with -the suffix `_parallel_{NUMBER}` (see teh example process chains in +the suffix `_parallel_{NUMBER}` (see the example process chains in `test_postbodies/parallel_persistent_processing.json`). So if you want to use a mapset in a later step, you have to pay attention to the naming of the mapset. diff --git a/setup.cfg b/setup.cfg index 6e25fbc..36c8197 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ name = actinia_parallel_plugin.wsgi description = actinia example plugin author = Anika Weinmann author-email = aweinmann@mundialis.de -license = GPU3 +license = GPL3 long-description = file: README.md long-description-content-type = text/x-rst; charset=UTF-8 url = https://github.com/pyscaffold/pyscaffold/ From d673652be605f5cec39b9fe9263a2067a0bf5dd8 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 08:34:03 +0200 Subject: [PATCH 11/47] tests are running --- .github/workflows/test.yml | 119 +++++++++++------- .gitignore | 1 + README.md | 14 ++- .../actinia-parallel-plugin-test/Dockerfile | 25 ++-- .../actinia-parallel-plugin-test.cfg | 5 +- .../run_integration_tests.sh | 3 + .../run_unittests.sh | 3 + docker/docker-compose-test.yml | 59 +++++++++ src/actinia_parallel_plugin/api/job.py | 4 +- src/actinia_parallel_plugin/core/jobs.py | 2 +- .../test_parallel_ephemeral_processing.py | 32 +++-- tests/test_resource_base.py | 29 +++++ 12 files changed, 218 insertions(+), 78 deletions(-) create mode 100644 docker/actinia-parallel-plugin-test/run_integration_tests.sh create mode 100644 docker/actinia-parallel-plugin-test/run_unittests.sh create mode 100644 docker/docker-compose-test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c186454..9aa0c07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,48 +9,81 @@ on: jobs: - unittests: + docker-compose: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - # with: - # path: "." - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Replace run only unittest command - run: | - sed -i "s+# RUN make test+RUN make unittest+g" docker/actinia-parallel-plugin-test/Dockerfile - - name: Unittests of actinia-parallel-plugin - id: docker_build - uses: docker/build-push-action@v2 - with: - push: false - tags: actinia-parallel-plugin-tests:alpine - context: . - file: docker/actinia-parallel-plugin-test/Dockerfile - no-cache: true - # pull: true - - integration-tests: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - # with: - # path: "." - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Replace run integration test command - run: | - sed -i "s+# RUN make test+RUN make integrationtest+g" docker/actinia-parallel-plugin-test/Dockerfile - - name: Integration tests of actinia-parallel-plugin - id: docker_build - uses: docker/build-push-action@v2 - with: - push: false - tags: actinia-parallel-plugin-test:alpine - context: . - file: docker/actinia-parallel-plugin-test/Dockerfile - no-cache: true - # pull: true + - uses: actions/checkout@v2 + + # - name: HELPEr + # run: ls -lah docker + + - name: Start containers + run: docker-compose -f "docker/docker-compose-test.yml" up -d --build + + - name: List running docker + run: docker ps + + - name: Run Tests + run: docker exec -it docker_actinia-test_1 sh + - name: Stop containers + run: docker-compose -f "docker/docker-compose-test.yml" down + + # - name: Install node + # uses: actions/setup-node@v1 + # with: + # node-version: 14.x + # + # - name: Install dependencies + # run: npm install + # + # - name: Run tests + # run: npm run test + # + # - name: Stop containers + # if: always() + # run: docker-compose -f "docker-compose.yml" down + # unittests: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # # with: + # # path: "." + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v1 + # - name: Replace run only unittest command + # run: | + # sed -i "s+# RUN make test+RUN make unittest+g" docker/actinia-parallel-plugin-test/Dockerfile + # - name: Unittests of actinia-parallel-plugin + # id: docker_build + # uses: docker/build-push-action@v2 + # with: + # push: false + # tags: actinia-parallel-plugin-tests:alpine + # context: . + # file: docker/actinia-parallel-plugin-test/Dockerfile + # no-cache: true + # # pull: true + # + # integration-tests: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # # with: + # # path: "." + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v1 + # - name: Replace run integration test command + # run: | + # sed -i "s+# RUN make test+RUN make integrationtest+g" docker/actinia-parallel-plugin-test/Dockerfile + # - name: Integration tests of actinia-parallel-plugin + # id: docker_build + # uses: docker/build-push-action@v2 + # with: + # push: false + # tags: actinia-parallel-plugin-test:alpine + # context: . + # file: docker/actinia-parallel-plugin-test/Dockerfile + # no-cache: true + # # pull: true diff --git a/.gitignore b/.gitignore index c45eb44..67ac5c7 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* +.coverage.* # Project files .ropeproject diff --git a/README.md b/README.md index 927b317..1497bc0 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,11 @@ rm -rf /usr/lib/python3.8/site-packages/actinia_parallel_plugin.wsgi-*.egg You can run the tests in the actinia test docker: ``` -docker build -f docker/actinia-parallel-plugin-test/Dockerfile -t actinia-parallel-plugin-test . -docker run -it actinia-parallel-plugin-test -i +docker-compose -f docker/docker-compose-test.yml build +docker-compose -f docker/docker-compose-test.yml up -d -cd /src/actinia-parallel-plugin/ +# exec docker and run tests manually +docker exec -it docker_actinia-test_1 sh # run all tests make test @@ -74,8 +75,11 @@ make unittest # run only integrationtests make integrationtest -# run only tests which are marked for development with the decorator '@pytest.mark.dev' -make devtest +# or run tests outside of docker container +docker exec -it docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh +docker exec -it docker_actinia-test_1 sh /usr/bin/run_unittests.sh + +docker-compose -f docker/docker-compose-test.yml down ``` ## Examples diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index c80ce73..b48cd61 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -9,19 +9,25 @@ ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia-parallel-plugin-test ENV DEFAULT_CONFIG_PATH /etc/default/actinia-parallel-plugin-test # install things only for tests -RUN apk add redis postgresql +# RUN apk add postgresql +# RUN apk add docker openrc +RUN apk add redis RUN pip3 install iniconfig colorlog pwgen +# RUN rc-update add docker boot +# RUN service docker start +# RUN docker pull kartoza/postgis + # COPY docker/actinia-parallel-plugin-test/start.sh /src/start.sh ENTRYPOINT ["/bin/sh"] CMD ["/src/start.sh"] -# # add data for tests -# RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_micro.zip && \ -# unzip nc_spm_08_micro.zip && \ -# rm -f nc_spm_08_micro.zip && \ -# mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08 +# add data for tests +RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_micro.zip && \ + unzip nc_spm_08_micro.zip && \ + rm -f nc_spm_08_micro.zip && \ + mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08 # RUN grass -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84 # copy needed files and configs for test @@ -34,4 +40,9 @@ WORKDIR /src/actinia-parallel-plugin/ RUN chmod a+x tests_with_redis.sh RUN make install -# RUN make test +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0 + +COPY docker/actinia-parallel-plugin-test/run_integration_tests.sh /usr/bin/run_integration_tests.sh +RUN chmod a+x /usr/bin/run_integration_tests.sh +COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh +RUN chmod a+x /usr/bin/run_unittests.sh diff --git a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg index 7dde08a..7dc4aa9 100644 --- a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg +++ b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg @@ -12,7 +12,8 @@ plugins = ["actinia_parallel_plugin"] force_https_urls = True [REDIS] -redis_server_url = localhost +redis_server_url = redis-test +redis_server_pw = pass redis_server_port = 6379 worker_queue_name = actinia_job worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log @@ -24,7 +25,7 @@ secret_key = token_signing_key_changeme save_interim_results = False [JOBTABLE] -host = postgis +host = postgis-test port = 5432 database = gis user = actinia diff --git a/docker/actinia-parallel-plugin-test/run_integration_tests.sh b/docker/actinia-parallel-plugin-test/run_integration_tests.sh new file mode 100644 index 0000000..d91a36c --- /dev/null +++ b/docker/actinia-parallel-plugin-test/run_integration_tests.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +make integrationtest diff --git a/docker/actinia-parallel-plugin-test/run_unittests.sh b/docker/actinia-parallel-plugin-test/run_unittests.sh new file mode 100644 index 0000000..dba0245 --- /dev/null +++ b/docker/actinia-parallel-plugin-test/run_unittests.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +make unittest diff --git a/docker/docker-compose-test.yml b/docker/docker-compose-test.yml new file mode 100644 index 0000000..b332ff1 --- /dev/null +++ b/docker/docker-compose-test.yml @@ -0,0 +1,59 @@ +version: "3" +services: + + actinia-test: + build: + context: .. + dockerfile: docker/actinia-parallel-plugin-test/Dockerfile + volumes: + - ..:/src/actinia-parallel-plugin/. + - /home/aweinmann/grassdata/:/actinia_core/grassdb/ + ports: + - "8088:8088" + environment: + - JOBTABLE_PW=actinia + depends_on: + - redis-test + - postgis-test + cap_add: + - SYS_PTRACE + networks: + - actinia-test + + redis-test: + image: redis:5.0.4-alpine + volumes: + - ./redis_data:/data + environment: + - REDIS_PASS_FILE=/data/config/.redis + command: [ + "sh", "-c", + ' + docker-entrypoint.sh + "/data/config/redis.conf" + --requirepass "$$(cat $$REDIS_PASS_FILE)" + ' + ] + ports: + - "6379:6379" + networks: + - actinia-test + + postgis-test: + image: mdillon/postgis:9.6-alpine + # network_mode: host + environment: + POSTGRES_USER: actinia + POSTGRES_PASSWORD: actinia + volumes: + - ./postgresql_init_data:/docker-entrypoint-initdb.d + # - ./postgresql_data:/var/lib/postgresql/data:Z + networks: + - actinia-test + + +networks: + actinia-test: + ipam: + config: + - subnet: 172.18.0.0/26 diff --git a/src/actinia_parallel_plugin/api/job.py b/src/actinia_parallel_plugin/api/job.py index e94a229..c3915a7 100644 --- a/src/actinia_parallel_plugin/api/job.py +++ b/src/actinia_parallel_plugin/api/job.py @@ -40,7 +40,7 @@ class JobId(Resource): """ Definition for endpoint standortsicherung - @app.route('/processes/standortsicherung/jobs/') + @app.route('/processing_parallel/jobs/') Contains HTTP GET endpoint reading a job Contains swagger documentation @@ -51,7 +51,7 @@ def get(self, jobid): """ Wrapper method to receive HTTP call and pass it to function This method is called by HTTP GET - @app.route('/processes/standortsicherung/jobs/') + @app.route('/processing_parallel/jobs/') This method is calling core method readJob """ if jobid is None: diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index ceb5677..fbb7978 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -92,7 +92,7 @@ def getJob(jobid): """ Method to read job from Jobtable by id This method can be called by HTTP GET - @app.route('/processes/standortsicherung/jobs/') + @app.route('/processing_parallel/jobs/') """ job, err = getJobById(jobid) diff --git a/tests/integrationtests/test_parallel_ephemeral_processing.py b/tests/integrationtests/test_parallel_ephemeral_processing.py index 940971c..fd18f8d 100644 --- a/tests/integrationtests/test_parallel_ephemeral_processing.py +++ b/tests/integrationtests/test_parallel_ephemeral_processing.py @@ -25,9 +25,7 @@ __maintainer__ = "mundialis GmbH % Co. KG" -import json import pytest -from flask import Response from ..test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX @@ -78,17 +76,9 @@ "list": [ { "module": "g.region", - "id": "g_region_2_parallel_block2", + "id": "g_region_1_parallel_block2", "inputs":[ - { - "import_descr": - { - "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", - "type": "raster" - }, - "param": "raster", - "value": "elev_ned_30m" - } + {"param": "raster", "value": "elevation@PERMANENT"} ], "flags": "p" }, @@ -96,7 +86,7 @@ "module": "r.univar", "id": "r_univar_2_parallel_block2", "inputs":[ - {"param": "map", "value": "elev_ned_30m"} + {"param": "map", "value": "elevation@PERMANENT"} ], "stdout": {"id": "stats", "format": "kv", "delimiter": "="}, "flags": "g" @@ -142,12 +132,18 @@ def test_post_parallel_ephemeral_processing(self): content_type=self.content_type, data=PC, ) - import pdb; pdb.set_trace() - resp = self.waitAsyncStatusAssertHTTP( + resp = self.waitAsyncBatchJob( rv, headers=self.user_auth_header, http_status=200, - status="finished", + status="SUCCESS", ) - assert "process_results" in resp, "No 'process_results' in response" - assert resp["process_results"] == ["grid1", "grid2", "grid3", "grid4"] + assert "actinia_core_response" in resp, \ + "No 'actinia_core_response' in response" + assert len(resp["actinia_core_response"]) == 4, \ + "There are not 4 actinia core responses" + process_results = [ + ac_resp["process_results"] for key, ac_resp in + resp["actinia_core_response"].items() if + ac_resp["process_results"] != {}] + assert "stats" in process_results[0] diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 0e771b0..baa8570 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -25,6 +25,7 @@ import signal import time +from flask.json import loads as json_loads from werkzeug.datastructures import Headers from actinia_core.testsuite import ActiniaTestCaseBase, URL_PREFIX @@ -135,3 +136,31 @@ def create_user(cls, name="guest", role="guest", cls.users_list.append(user) return name, group, cls.auth_header[role] + + def waitAsyncBatchJob(self, rv, headers, http_status=200, + status="SUCCESS", message_check=None): + resp_data = json_loads(rv.data) + batchid = resp_data["actinia_gdi_batchid"] + + while True: + rv = self.server.get( + URL_PREFIX + "/processing_parallel/batchjobs/%s" % (batchid), + headers=headers + ) + resp_data = json_loads(rv.data) + if (resp_data["status"] == "SUCCESS" + or resp_data["status"] == "ERROR" + or resp_data["status"] == "TERMINATED"): + break + + time.sleep(0.2) + self.assertEqual(resp_data["status"], status) + self.assertEqual(rv.status_code, http_status, + "HTML status code is wrong %i" % rv.status_code) + + if message_check is not None: + self.assertTrue(message_check in resp_data["message"], + (f"Message is {resp_data['message']}")) + + time.sleep(0.4) + return resp_data From 5bb07c2a1bd60e4e9fb85fc087967bf2599883e5 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 10:07:14 +0200 Subject: [PATCH 12/47] unittests --- .github/workflows/test.yml | 91 ++--- README.md | 14 + .../actinia-parallel-plugin-test/Dockerfile | 7 +- tests/unittests/test_batches.py | 358 ++++++++++++++++++ tests_with_redis.sh | 34 -- 5 files changed, 402 insertions(+), 102 deletions(-) create mode 100644 tests/unittests/test_batches.py delete mode 100644 tests_with_redis.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9aa0c07..6e5aea7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,81 +9,42 @@ on: jobs: - docker-compose: + integration-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - # - name: HELPEr - # run: ls -lah docker - - name: Start containers run: docker-compose -f "docker/docker-compose-test.yml" up -d --build - name: List running docker run: docker ps - - name: Run Tests - run: docker exec -it docker_actinia-test_1 sh + - name: Run Unittests + run: docker exec -it docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh + - name: Stop containers run: docker-compose -f "docker/docker-compose-test.yml" down - # - name: Install node - # uses: actions/setup-node@v1 - # with: - # node-version: 14.x - # - # - name: Install dependencies - # run: npm install - # - # - name: Run tests - # run: npm run test - # - # - name: Stop containers - # if: always() - # run: docker-compose -f "docker-compose.yml" down - # unittests: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v2 - # # with: - # # path: "." - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v1 - # - name: Replace run only unittest command - # run: | - # sed -i "s+# RUN make test+RUN make unittest+g" docker/actinia-parallel-plugin-test/Dockerfile - # - name: Unittests of actinia-parallel-plugin - # id: docker_build - # uses: docker/build-push-action@v2 - # with: - # push: false - # tags: actinia-parallel-plugin-tests:alpine - # context: . - # file: docker/actinia-parallel-plugin-test/Dockerfile - # no-cache: true - # # pull: true - # - # integration-tests: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v2 - # # with: - # # path: "." - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v1 - # - name: Replace run integration test command - # run: | - # sed -i "s+# RUN make test+RUN make integrationtest+g" docker/actinia-parallel-plugin-test/Dockerfile - # - name: Integration tests of actinia-parallel-plugin - # id: docker_build - # uses: docker/build-push-action@v2 - # with: - # push: false - # tags: actinia-parallel-plugin-test:alpine - # context: . - # file: docker/actinia-parallel-plugin-test/Dockerfile - # no-cache: true - # # pull: true + unittests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + # with: + # path: "." + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Replace run only unittest command + run: | + sed -i "s+# RUN make test+RUN make unittest+g" docker/actinia-parallel-plugin-test/Dockerfile + - name: Unittests of actinia-parallel-plugin + id: docker_build + uses: docker/build-push-action@v2 + with: + push: false + tags: actinia-parallel-plugin-tests:alpine + context: . + file: docker/actinia-parallel-plugin-test/Dockerfile + no-cache: true + # pull: true diff --git a/README.md b/README.md index 1497bc0..3c412ea 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,20 @@ docker exec -it docker_actinia-test_1 sh /usr/bin/run_unittests.sh docker-compose -f docker/docker-compose-test.yml down ``` +You can also run the tests in the GHA workflows locally via [act](https://github.com/nektos/act). +To run docker-compose inside a workflow [act_base](https://github.com/lucasctrl/act_base) can be used. +With these you can run the following to run the tests: +``` +# list all workflows +act -l + +# run workflow +act -j integration-tests -P ubuntu-latest=lucasalt/act_base:latest +act -j unittests -P ubuntu-latest=lucasalt/act_base:latest +``` + + + ## Examples ### Requesting batch job and job endpoints diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index b48cd61..3d2f942 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -37,12 +37,13 @@ COPY . /src/actinia-parallel-plugin/ WORKDIR /src/actinia-parallel-plugin/ -RUN chmod a+x tests_with_redis.sh RUN make install ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0 COPY docker/actinia-parallel-plugin-test/run_integration_tests.sh /usr/bin/run_integration_tests.sh RUN chmod a+x /usr/bin/run_integration_tests.sh -COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh -RUN chmod a+x /usr/bin/run_unittests.sh +# COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh +# RUN chmod a+x /usr/bin/run_unittests.sh + +# RUN make test diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py new file mode 100644 index 0000000..82f1237 --- /dev/null +++ b/tests/unittests/test_batches.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Unit tests for core functionallity of batches +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import pytest +import datetime +from actinia_parallel_plugin.core.batches import checkProcessingBlockFinished + + +baseurl = "http://localhost:8088/api/v3" +resource_id1 = "resource_id-a6da5e00-d2a3-4804-b82f-e03f92ab1cd4" +resource_id2 = "resource_id-291e8428-ec86-40a6-9ed2-ef1e14357aff" +jobs = [ + { + "idpk_jobs": 7, + "process": "ephemeral", + "rule_configuration": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "parallel": "true", + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "job_description": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 930771), + "time_started": None, + "time_estimated": None, + "time_ended": None, + "status": "PREPARING", + "actinia_core_response": None, + "actinia_core_jobid": None, + "actinia_core_url": None, + "actinia_core_platform": None, + "actinia_core_platform_name": None, + "terraformer_id": None, + "terraformer_response": None, + "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", + "message": None, + "batch_id": 2, + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + { + "idpk_jobs": 8, + "process": "ephemeral", + "rule_configuration": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "parallel": "false", + "processing_block": 3, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "job_description": { + "list": [ + { + "id": "g_region_nonparallel_block3", + "flags": "p", + "inputs": [ + {"param": "raster", "value": "elevation@PERMANENT"} + ], + "module": "g.region", + "outputs": [], + } + ], + "version": "1", + "batch_id": 2, + "processing_block": 3, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 940472), + "time_started": None, + "time_estimated": None, + "time_ended": None, + "status": "PREPARING", + "actinia_core_response": None, + "actinia_core_jobid": None, + "actinia_core_url": None, + "actinia_core_platform": None, + "actinia_core_platform_name": None, + "terraformer_id": None, + "terraformer_response": None, + "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", + "message": None, + "batch_id": 2, + "processing_block": 3, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + { + "idpk_jobs": 5, + "process": "ephemeral", + "rule_configuration": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "parallel": "false", + "processing_block": 1, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "job_description": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "processing_block": 1, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 906827), + "time_started": None, + "time_estimated": None, + "time_ended": datetime.datetime(2022, 6, 2, 7, 20, 15), + "status": "SUCCESS", + "actinia_core_response": { + "urls": { + "status": f"{baseurl}/resources/actinia-gdi/{resource_id2}", + "resources": [], + }, + "status": "finished", + "message": "Processing successfully finished", + "user_id": "actinia-gdi", + "api_info": { + "path": "/api/v3/locations/nc_spm_08_grass7_root/" + "processing_parallel", + "method": "POST", + "endpoint": "asyncparallelephermeralresource", + "post_url": f"{baseurl}/locations/nc_spm_08_grass7_root/" + "processing_parallel", + "request_url": f"{baseurl}/locations/nc_spm_08_grass7_root/" + "processing_parallel", + }, + "datetime": "2022-06-02 07:20:15.942998", + "progress": {"step": 2, "num_of_steps": 2}, + "http_code": 200, + "timestamp": 1654154415.9429944, + "time_delta": 0.9949047565460205, + "process_log": [{"..."}], + "resource_id": resource_id2, + "accept_datetime": "2022-06-02 07:20:14.948113", + "process_results": {}, + "accept_timestamp": 1654154414.9481106, + "process_chain_list": [ + { + "list": [{"..."}], + "version": "1", + } + ], + }, + "actinia_core_jobid": resource_id2, + "actinia_core_url": None, + "actinia_core_platform": None, + "actinia_core_platform_name": None, + "terraformer_id": None, + "terraformer_response": None, + "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", + "message": None, + "batch_id": 2, + "processing_block": 1, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + { + "idpk_jobs": 6, + "process": "ephemeral", + "rule_configuration": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "parallel": "true", + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "job_description": { + "list": [{"..."}], + "version": "1", + "batch_id": 2, + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, + "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 920875), + "time_started": None, + "time_estimated": None, + "time_ended": datetime.datetime(2022, 6, 2, 7, 20, 49), + "status": "SUCCESS", + "actinia_core_response": { + "urls": { + "status": f"{baseurl}/resources/actinia-gdi/{resource_id1}", + "resources": [], + }, + "status": "finished", + "message": "Processing successfully finished", + "user_id": "actinia-gdi", + "api_info": { + "path": "/api/v3/locations/nc_spm_08_grass7_root/" + "processing_parallel", + "method": "POST", + "endpoint": "asyncparallelephermeralresource", + "post_url": f"{baseurl}/locations/nc_spm_08_grass7_root/" + "processing_parallel", + "request_url": f"{baseurl}/locations/nc_spm_08_grass7_root/" + "processing_parallel", + }, + "datetime": "2022-06-02 07:20:49.262223", + "progress": {"step": 2, "num_of_steps": 2}, + "http_code": 200, + "timestamp": 1654154449.2622168, + "time_delta": 0.4729771614074707, + "process_log": [{"..."}], + "resource_id": resource_id1, + "accept_datetime": "2022-06-02 07:20:48.789271", + "process_results": {}, + "accept_timestamp": 1654154448.7892694, + "process_chain_list": [ + { + "list": [{"..."}], + "version": "1", + } + ], + }, + "actinia_core_jobid": resource_id1, + "actinia_core_url": None, + "actinia_core_platform": None, + "actinia_core_platform_name": None, + "terraformer_id": None, + "terraformer_response": None, + "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", + "message": None, + "batch_id": 2, + "processing_block": 2, + "batch_description": { + "jobs": [ + {"list": [{"..."}], "version": "1", "parallel": "false"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "true"}, + {"list": [{"..."}], "version": "1", "parallel": "false"}, + ] + }, + }, +] +block = [1, 2, 3] +ref_out = [True, False, False] + + +@pytest.mark.unittest +@pytest.mark.parametrize( + "block,ref_out", + [(block[0], ref_out[0]), (block[1], ref_out[1]), (block[2], ref_out[2])], +) +def test_checkProcessingBlockFinished(block, ref_out): + """Test for checkProcessingBlockFinished function.""" + + out = checkProcessingBlockFinished(jobs, block) + assert ( + out is ref_out + ), f"Wrong result from transform_input for block {block}" diff --git a/tests_with_redis.sh b/tests_with_redis.sh deleted file mode 100644 index 323860d..0000000 --- a/tests_with_redis.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env sh - -# start redis server -redis-server & -sleep 1 -redis-cli ping - -# TODO start postgresql - -# start webhook server -webhook-server --host "0.0.0.0" --port "5005" & -sleep 10 - -# run tests -echo $ACTINIA_CUSTOM_TEST_CFG -echo $DEFAULT_CONFIG_PATH - -if [ "$1" == "dev" ] -then - echo "Executing only 'dev' tests ..." - python3 setup.py test --addopts "-m dev" -elif [ "$1" == "integrationtest" ] -then - python3 setup.py test --addopts "-m 'integrationtest'" -else - python3 setup.py test -fi - -TEST_RES=$? - -# stop redis server -redis-cli shutdown - -return $TEST_RES From 9364924262c99e618ff8123f3503087b7beed128 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 10:52:12 +0200 Subject: [PATCH 13/47] small changes --- Makefile | 6 +++--- README.md | 1 - docker/actinia-parallel-plugin-test/Dockerfile | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4facc69..0da753a 100644 --- a/Makefile +++ b/Makefile @@ -19,13 +19,13 @@ dist: python3 setup.py dist test: - ./tests_with_redis.sh + python3 setup.py test unittest: python3 setup.py test --addopts "-m unittest" devtest: - ./tests_with_redis.sh dev + python3 setup.py test --addopts "-m 'dev'" integrationtest: - ./tests_with_redis.sh integrationtest + python3 setup.py test --addopts "-m 'integrationtest'" diff --git a/README.md b/README.md index 3c412ea..3747977 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,6 @@ act -j unittests -P ubuntu-latest=lucasalt/act_base:latest ``` - ## Examples ### Requesting batch job and job endpoints diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index 3d2f942..763608c 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -43,7 +43,5 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0 COPY docker/actinia-parallel-plugin-test/run_integration_tests.sh /usr/bin/run_integration_tests.sh RUN chmod a+x /usr/bin/run_integration_tests.sh -# COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh -# RUN chmod a+x /usr/bin/run_unittests.sh # RUN make test From 91d3c00124d6a50d1db348b8765c1d6c2420d7af Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:09:10 +0200 Subject: [PATCH 14/47] remove persistent processing --- .github/workflows/test.yml | 2 +- .../api/parallel_processing.py | 121 --------- .../core/persistent_processing.py | 232 ------------------ 3 files changed, 1 insertion(+), 354 deletions(-) delete mode 100644 src/actinia_parallel_plugin/api/parallel_processing.py delete mode 100644 src/actinia_parallel_plugin/core/persistent_processing.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e5aea7..6ddb30f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: List running docker run: docker ps - - name: Run Unittests + - name: Run integration test run: docker exec -it docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh - name: Stop containers diff --git a/src/actinia_parallel_plugin/api/parallel_processing.py b/src/actinia_parallel_plugin/api/parallel_processing.py deleted file mode 100644 index 5b57863..0000000 --- a/src/actinia_parallel_plugin/api/parallel_processing.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-present mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Parallel processing -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - -from flask import request, make_response, jsonify, g -from flask_restful_swagger_2 import swagger, Resource - -from actinia_api import URL_PREFIX -from actinia_core.models.response_models import \ - SimpleResponseModel - -from actinia_parallel_plugin.apidocs import helloworld -from actinia_parallel_plugin.core.batches import ( - createBatch, - createBatchId, - createBatchResponseDict, - getJobsByBatchId, - startProcessingBlock, -) -from actinia_parallel_plugin.model.response_models import ( - SimpleStatusCodeResponseModel, -) -from actinia_parallel_plugin.resources.logging import log - - -class AsyncParallelPersistentResource(Resource): - """Resource for parallel processing""" - - def __init__(self): - super(AsyncParallelPersistentResource, self).__init__() - self.location_name = None - self.mapset_name = None - self.batch_id = None - - @swagger.doc(helloworld.describeHelloWorld_get_docs) - def get(self, location_name, mapset_name): - """Get 'Hello world!' as answer string.""" - return SimpleStatusCodeResponseModel(status=200, message="TEST") - - # TODO get all batch jobs - @swagger.doc(helloworld.describeHelloWorld_get_docs) - def post(self, location_name, mapset_name): - """Persistent parallel processing.""" - - self.location_name = location_name - self.mapset_name = mapset_name - self.post_url = self.api_info["request_url"] - - json_dict = request.get_json(force=True) - log.info("Received HTTP POST with batchjob: %s" % - str(json_dict)) - - # assign new batchid - self.batch_id = createBatchId() - # create processing blocks and insert jobs into jobtable - jobs_in_db = createBatch(json_dict, "persistent", self.batch_id) - if jobs_in_db is None: - res = (jsonify(SimpleResponseModel( - status=500, - message=('Error: Batch Processing Chain JSON has no ' - 'jobs.') - ))) - return make_response(res, 500) - - # Generate the base of the status URL - self.base_status_url = f"{request.host_url}{URL_PREFIX}/resources/" \ - f"{g.user.user_id}/" - - # start first processing block - first_jobs = startProcessingBlock( - jobs_in_db, - 1, - self.batch_id, - self.location_name, - g.user, - request.url, - self.post_url, - request.endpoint, - request.method, - request.path, - self.base_status_url, - self.mapset_name, - "persistent" - ) - first_status = [entry["status"] for entry in first_jobs] - all_jobs = getJobsByBatchId(self.batch_id, "persistent") - if None in first_jobs: - res = (jsonify(SimpleResponseModel( - status=500, - message=('Error: There was a problem starting the ' - 'first jobs of the batchjob.') - ))) - return make_response(res, 500) - elif "ERROR" not in first_status: - return make_response(jsonify(createBatchResponseDict(all_jobs)), - 201) - else: - return make_response(jsonify(createBatchResponseDict(all_jobs)), - 412) diff --git a/src/actinia_parallel_plugin/core/persistent_processing.py b/src/actinia_parallel_plugin/core/persistent_processing.py deleted file mode 100644 index 40ca868..0000000 --- a/src/actinia_parallel_plugin/core/persistent_processing.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2022 mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Parallel processing -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - - -import sys -import traceback -import pickle - -from actinia_core.processing.actinia_processing.ephemeral.\ - persistent_processing import PersistentProcessing -from actinia_core.core.common.exceptions \ - import AsyncProcessError, AsyncProcessTermination -from actinia_core.core.common.exceptions import AsyncProcessTimeLimit -from actinia_core.models.response_models \ - import ExceptionTracebackModel - -from actinia_parallel_plugin.core.batches import ( - checkProcessingBlockFinished, - # createBatchResponseDict, - getJobsByBatchId, - startProcessingBlock, -) -from actinia_parallel_plugin.core.jobs import updateJob - - -class ParallelPersistentProcessing(PersistentProcessing): - - def __init__(self, rdc, batch_id, processing_block, jobid, - user, request_url, post_url, endpoint, method, path, - base_status_url): - super(ParallelPersistentProcessing, self).__init__(rdc) - self.batch_id = batch_id - self.processing_block = processing_block - self.jobid = jobid - self.post_url = post_url - self.user = user - self.request_url = request_url - self.post_url = post_url - self.endpoint = endpoint - self.method = method - self.path = path - self.base_status_url = base_status_url - - # def _execute(self, process_chain, skip_permission_check=False): - def _execute(self, skip_permission_check=False): - """Overwrite this function in subclasses. - - This function will be executed by the run() function - - - Setup logger and credentials - - Analyse the process chain - - Create the temporal database - - Initialize the GRASS environment and create the temporary mapset - - Run the modules - - Parse the stdout output of the modules and generate the module - results - - Args: - skip_permission_check (bool): If set True, the permission checks of - module access and process num - limits are not performed - - Raises: - This method will raise an AsyncProcessError, AsyncProcessTimeLimit - or AsyncProcessTermination - - """ - # Create the process chain - if self.rdc.iteration is not None: - process_list = \ - self._create_temporary_grass_environment_and_process_list_for_iteration( - skip_permission_check=skip_permission_check) - else: - process_list = self._create_temporary_grass_environment_and_process_list( - skip_permission_check=skip_permission_check) - - # Run all executables - self._execute_process_list(process_list=process_list) - # Parse the module sdtout outputs and create the results - self._parse_module_outputs() - - def run(self): - """This function will run the processing and will catch and process - any Exceptions that were raised while processing. Call this function - to run the processing. - - You have to implement/overwrite two methods that are called here: - - * self._execute() - * self._final_cleanup() - - e_type, e_value, e_traceback = sys.exc_info() - message = [e.__class__, e_type, e_value, traceback.format_tb( - e_traceback)] - message = pprint.pformat(message) - """ - try: - # Run the _execute function that does all the work - self._execute() - except AsyncProcessTermination as e: - self.run_state = {"terminated": str(e)} - except AsyncProcessTimeLimit as e: - self.run_state = {"time limit exceeded": str(e)} - except AsyncProcessError as e: - e_type, e_value, e_tb = sys.exc_info() - model = ExceptionTracebackModel( - message=str(e_value), - traceback=traceback.format_tb(e_tb), - type=str(e_type) - ) - self.run_state = {"error": str(e), "exception": model} - except KeyboardInterrupt as e: - e_type, e_value, e_tb = sys.exc_info() - model = ExceptionTracebackModel( - message=str(e_value), - traceback=traceback.format_tb(e_tb), - type=str(e_type) - ) - self.run_state = {"error": str(e), "exception": model} - except Exception as e: - e_type, e_value, e_tb = sys.exc_info() - model = ExceptionTracebackModel( - message=str(e_value), - traceback=traceback.format_tb(e_tb), - type=str(e_type) - ) - self.run_state = {"error": str(e), "exception": model} - finally: - try: - # Call the final cleanup, before sending the status messages - self._final_cleanup() - except Exception as e: - e_type, e_value, e_tb = sys.exc_info() - model = ExceptionTracebackModel( - message=str(e_value), - traceback=traceback.format_tb(e_tb), - type=str(e_type) - ) - self.run_state = {"error": str(e), "exception": model} - # After all processing finished, send the final status - if "success" in self.run_state: - self._send_resource_finished(message=self.finish_message, - results=self.module_results) - elif "terminated" in self.run_state: - # Send an error message if an exception was raised - self._send_resource_terminated( - message=self.run_state["terminated"]) - elif "time limit exceeded" in self.run_state: - self._send_resource_time_limit_exceeded( - message=self.run_state["time limit exceeded"]) - elif "error" in self.run_state: - # Send an error message if an exception was raised - self._send_resource_error( - message=self.run_state["error"], - exception=self.run_state["exception"]) - else: - self._send_resource_error(message="Unknown error") - self._update_and_check_batch_jobs() - - def _update_and_check_batch_jobs(self): - """Checks batch jobs and starts new batch block if the current block - is successfully finished. - """ - - # update job to finished - resource_id = self.resource_id - response_data = self.resource_logger.get( - self.user_id, self.resource_id) - _, response_model = pickle.loads(response_data) - updateJob(resource_id, response_model, self.jobid) - - if "finished" == response_model["status"]: - jobs_from_batch = getJobsByBatchId( - self.batch_id, - "persistent" - ) - all_blocks = [ - job["processing_block"] for job in jobs_from_batch] - block = int(self.processing_block) - block_done = checkProcessingBlockFinished( - jobs_from_batch, block) - if block_done is True and block < max(all_blocks): - next_block = block + 1 - startProcessingBlock( - jobs_from_batch, - next_block, - self.batch_id, - self.location_name, - self.mapset_name, - self.user, - self.request_url, - self.post_url, - self.endpoint, - self.method, - self.path, - self.base_status_url, - "persistent" - ) - - elif (response_model["status"] == "error" or - response_model["status"] == "terminated"): - # In this case, nothing happens and the next block is not - # started. - pass - - -def start_job(*args): - processing = ParallelPersistentProcessing(*args) - processing.run() From 34a1503e3b95d529d723e78a52b91e703e2438aa Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:14:13 +0200 Subject: [PATCH 15/47] update readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 3747977..9747f30 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,7 @@ Attention: createBatch is designed as an ephemeral process. TODOs: -* Test of exporters in PC -* APIDOCS -* Tests +* exporter in PC * using stdout/export in PC of next block #### Persistent processing From 87d6e6287890ef6a1b8e22136f4ebcf401f7f8b5 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:23:24 +0200 Subject: [PATCH 16/47] try to fix test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ddb30f..4d79d57 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: run: docker ps - name: Run integration test - run: docker exec -it docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh + run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh - name: Stop containers run: docker-compose -f "docker/docker-compose-test.yml" down From 1b5ba6bf22281536e7b05a009d9fe03e0d17d5f1 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:36:12 +0200 Subject: [PATCH 17/47] more logs by different status --- tests/test_resource_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index baa8570..02e8110 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -154,7 +154,8 @@ def waitAsyncBatchJob(self, rv, headers, http_status=200, break time.sleep(0.2) - self.assertEqual(resp_data["status"], status) + self.assertEqual(resp_data["status"], status, \ + f"Process has not status '{status}': {resp_data}") self.assertEqual(rv.status_code, http_status, "HTML status code is wrong %i" % rv.status_code) From c8c4ed942690c3b7017ecfc0460c9d6586133595 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:43:16 +0200 Subject: [PATCH 18/47] try to get more infos of test failure --- tests/test_resource_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 02e8110..6425eaa 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -154,7 +154,9 @@ def waitAsyncBatchJob(self, rv, headers, http_status=200, break time.sleep(0.2) - self.assertEqual(resp_data["status"], status, \ + print(resp_data) + self.assertEqual( + resp_data["status"], status, f"Process has not status '{status}': {resp_data}") self.assertEqual(rv.status_code, http_status, "HTML status code is wrong %i" % rv.status_code) From e80110e83aa39a9865cc40c3b0a3172adc825828 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 11:59:22 +0200 Subject: [PATCH 19/47] tests --- docker/actinia-parallel-plugin-test/run_integration_tests.sh | 2 +- docker/actinia-parallel-plugin-test/run_unittests.sh | 2 +- tests/test_resource_base.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/actinia-parallel-plugin-test/run_integration_tests.sh b/docker/actinia-parallel-plugin-test/run_integration_tests.sh index d91a36c..2073cff 100644 --- a/docker/actinia-parallel-plugin-test/run_integration_tests.sh +++ b/docker/actinia-parallel-plugin-test/run_integration_tests.sh @@ -1,3 +1,3 @@ #!/bin/bash -make integrationtest +python3 setup.py test --addopts "-m 'integrationtest'" diff --git a/docker/actinia-parallel-plugin-test/run_unittests.sh b/docker/actinia-parallel-plugin-test/run_unittests.sh index dba0245..1523ade 100644 --- a/docker/actinia-parallel-plugin-test/run_unittests.sh +++ b/docker/actinia-parallel-plugin-test/run_unittests.sh @@ -1,3 +1,3 @@ #!/bin/bash -make unittest +python3 setup.py test --addopts "-m 'unittest'" diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 6425eaa..3d50ca2 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -154,7 +154,6 @@ def waitAsyncBatchJob(self, rv, headers, http_status=200, break time.sleep(0.2) - print(resp_data) self.assertEqual( resp_data["status"], status, f"Process has not status '{status}': {resp_data}") From f41a9ea746df1728dfa9f5721d2e45e7a1480047 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 12:02:37 +0200 Subject: [PATCH 20/47] tests --- tests/test_resource_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 3d50ca2..fc8e54e 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -156,7 +156,7 @@ def waitAsyncBatchJob(self, rv, headers, http_status=200, time.sleep(0.2) self.assertEqual( resp_data["status"], status, - f"Process has not status '{status}': {resp_data}") + msg=f"Process has not status '{status}': {resp_data}") self.assertEqual(rv.status_code, http_status, "HTML status code is wrong %i" % rv.status_code) From b77711efd14176fd879ca44ae9c64a9b37f67bfc Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 12:19:15 +0200 Subject: [PATCH 21/47] test --- docker/docker-compose-test.yml | 15 +-------------- docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/docker/docker-compose-test.yml b/docker/docker-compose-test.yml index b332ff1..59411e1 100644 --- a/docker/docker-compose-test.yml +++ b/docker/docker-compose-test.yml @@ -17,8 +17,6 @@ services: - postgis-test cap_add: - SYS_PTRACE - networks: - - actinia-test redis-test: image: redis:5.0.4-alpine @@ -36,11 +34,9 @@ services: ] ports: - "6379:6379" - networks: - - actinia-test postgis-test: - image: mdillon/postgis:9.6-alpine + image: postgis/postgis:14-3.2-alpine # network_mode: host environment: POSTGRES_USER: actinia @@ -48,12 +44,3 @@ services: volumes: - ./postgresql_init_data:/docker-entrypoint-initdb.d # - ./postgresql_data:/var/lib/postgresql/data:Z - networks: - - actinia-test - - -networks: - actinia-test: - ipam: - config: - - subnet: 172.18.0.0/26 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 99b9664..0ae55de 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -39,7 +39,7 @@ services: - actinia postgis: - image: mdillon/postgis:9.6-alpine + image: postgis/postgis:14-3.2-alpine # network_mode: host environment: POSTGRES_USER: actinia From b73747393fdfa6e6c1d980b385c0ad065349cd3e Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 12:29:51 +0200 Subject: [PATCH 22/47] docker logs in tests --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d79d57..43f0358 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,9 @@ jobs: - name: List running docker run: docker ps + - name: Docker logs + run: docker logs docker_actinia-test_1 + - name: Run integration test run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh From da8f3b9ab0f93a91e06cb01f511eafc759d0758c Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 12:34:02 +0200 Subject: [PATCH 23/47] docker logs in tests --- docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0ae55de..fdb68bd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,6 +7,7 @@ services: dockerfile: docker/Dockerfile volumes: - ..:/src/actinia-parallel-plugin/. + - /home/aweinmann/grassdata/:/actinia_core/grassdb/ ports: - "8088:8088" environment: From c42af090acec6e29403a99242e6b284f273f5d9f Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 12:36:50 +0200 Subject: [PATCH 24/47] docker logs in tests --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43f0358..9a893dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,9 +20,12 @@ jobs: - name: List running docker run: docker ps - - name: Docker logs + - name: Docker logs actinia run: docker logs docker_actinia-test_1 + - name: Docker logs postgis + run: docker logs docker_postgis-test_1 + - name: Run integration test run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh From ed881b4dd26a55b58fe35c0387e677328ff35c8a Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 21:09:50 +0200 Subject: [PATCH 25/47] test running locally --- .github/workflows/test.yml | 3 +- .../actinia-parallel-plugin-test/Dockerfile | 2 + docker/docker-compose-test.yml | 13 +++ docker/docker-compose.yml | 2 +- tests/integrationtests/test_00_connections.py | 80 +++++++++++++++++++ 5 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 tests/integrationtests/test_00_connections.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a893dc..8c5f20f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,8 @@ jobs: run: docker logs docker_postgis-test_1 - name: Run integration test - run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh + run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_unittests.sh + # run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh - name: Stop containers run: docker-compose -f "docker/docker-compose-test.yml" down diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index 763608c..3fbb270 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -43,5 +43,7 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0 COPY docker/actinia-parallel-plugin-test/run_integration_tests.sh /usr/bin/run_integration_tests.sh RUN chmod a+x /usr/bin/run_integration_tests.sh +COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh +RUN chmod a+x /usr/bin/run_unittests.sh # RUN make test diff --git a/docker/docker-compose-test.yml b/docker/docker-compose-test.yml index 59411e1..a617a3e 100644 --- a/docker/docker-compose-test.yml +++ b/docker/docker-compose-test.yml @@ -17,6 +17,8 @@ services: - postgis-test cap_add: - SYS_PTRACE + networks: + - actinia-test redis-test: image: redis:5.0.4-alpine @@ -34,6 +36,8 @@ services: ] ports: - "6379:6379" + networks: + - actinia-test postgis-test: image: postgis/postgis:14-3.2-alpine @@ -44,3 +48,12 @@ services: volumes: - ./postgresql_init_data:/docker-entrypoint-initdb.d # - ./postgresql_data:/var/lib/postgresql/data:Z + networks: + - actinia-test + + +networks: + actinia-test: + ipam: + config: + - subnet: 172.18.0.0/26 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fdb68bd..f689df4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,7 +7,7 @@ services: dockerfile: docker/Dockerfile volumes: - ..:/src/actinia-parallel-plugin/. - - /home/aweinmann/grassdata/:/actinia_core/grassdb/ + # - /home/.../grassdata/:/actinia_core/grassdb/ ports: - "8088:8088" environment: diff --git a/tests/integrationtests/test_00_connections.py b/tests/integrationtests/test_00_connections.py new file mode 100644 index 0000000..087fe5a --- /dev/null +++ b/tests/integrationtests/test_00_connections.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2022 mundialis GmbH & Co. KG + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Parallel ephemeral processing tests +""" + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH % Co. KG" + +import os +import pytest +import redis +import psycopg2 + +# from actinia_core.core.common.redis_base import RedisBaseInterface +from actinia_core.core.common.config import global_config as config + +from actinia_parallel_plugin.resources.config import JOBTABLE + +from ..test_resource_base import ActiniaResourceTestCaseBase#, URL_PREFIX + + +class ConnectionTest(ActiniaResourceTestCaseBase): + + @pytest.mark.integrationtest + def test_redis_connection(self): + """Test redis connection + """ + if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: + config.read(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) + kwargs = dict() + kwargs["host"] = config.REDIS_SERVER_URL + kwargs["port"] = config.REDIS_SERVER_PORT + if config.REDIS_SERVER_PW and config.REDIS_SERVER_PW is not None: + kwargs["password"] = config.REDIS_SERVER_PW + connection_pool = redis.ConnectionPool(**kwargs) + redis_server = redis.StrictRedis(connection_pool=connection_pool) + try: + redis_server.ping() + assert True + except redis.exceptions.ResponseError: + assert False, "Could not connect to redis ({kwargs['host']})" + except redis.exceptions.AuthenticationError: + assert False, "Invalid redis password" + except redis.exceptions.ConnectionError as e: + assert False, f"Redis connection error: {e}" + connection_pool.disconnect() + + @pytest.mark.integrationtest + def test_postgis_connection(self): + """Test postgis connection + """ + try: + conn = psycopg2.connect(**{ + 'host': JOBTABLE.host, + 'port': JOBTABLE.port, + 'user': JOBTABLE.user, + 'password': JOBTABLE.pw + }) + assert True + conn.close() + except Exception: + assert False, "Postgis connection failed!" From 10f3ffef0c78463d95b87617840bde2d5d1db039 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 21:12:28 +0200 Subject: [PATCH 26/47] linting --- tests/integrationtests/test_00_connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationtests/test_00_connections.py b/tests/integrationtests/test_00_connections.py index 087fe5a..f232b69 100644 --- a/tests/integrationtests/test_00_connections.py +++ b/tests/integrationtests/test_00_connections.py @@ -34,7 +34,7 @@ from actinia_parallel_plugin.resources.config import JOBTABLE -from ..test_resource_base import ActiniaResourceTestCaseBase#, URL_PREFIX +from ..test_resource_base import ActiniaResourceTestCaseBase class ConnectionTest(ActiniaResourceTestCaseBase): From 0b09228c7d1ad140384a9e2a55dab9ffc31e1f2f Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 2 Jun 2022 21:18:04 +0200 Subject: [PATCH 27/47] linting --- tests/integrationtests/test_00_connections.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrationtests/test_00_connections.py b/tests/integrationtests/test_00_connections.py index f232b69..7ce4ce0 100644 --- a/tests/integrationtests/test_00_connections.py +++ b/tests/integrationtests/test_00_connections.py @@ -29,7 +29,6 @@ import redis import psycopg2 -# from actinia_core.core.common.redis_base import RedisBaseInterface from actinia_core.core.common.config import global_config as config from actinia_parallel_plugin.resources.config import JOBTABLE From bfd6141c6667e76517d93998bfa171ee5624efba Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 3 Jun 2022 08:10:22 +0200 Subject: [PATCH 28/47] remove volume --- docker/docker-compose-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose-test.yml b/docker/docker-compose-test.yml index a617a3e..2e754a4 100644 --- a/docker/docker-compose-test.yml +++ b/docker/docker-compose-test.yml @@ -7,7 +7,6 @@ services: dockerfile: docker/actinia-parallel-plugin-test/Dockerfile volumes: - ..:/src/actinia-parallel-plugin/. - - /home/aweinmann/grassdata/:/actinia_core/grassdb/ ports: - "8088:8088" environment: From f3f60b389ea5acf9bf38792dc0167dcf0b0e3ac6 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 8 Jun 2022 08:38:41 +0200 Subject: [PATCH 29/47] CT review 1 --- README.md | 14 -------- .../actinia-parallel-plugin-test/Dockerfile | 9 ----- src/actinia_parallel_plugin/apidocs/jobs.py | 1 - .../model/response_models.py | 33 ------------------- 4 files changed, 57 deletions(-) diff --git a/README.md b/README.md index 9747f30..a29f9ab 100644 --- a/README.md +++ b/README.md @@ -119,17 +119,3 @@ Attention: TODOs: * exporter in PC * using stdout/export in PC of next block - -#### Persistent processing -You can also start a **persistent** batch job via: -``` -# parallel persistent processing (TODO!!!) -curl -u actinia-gdi:actinia-gdi -X POST -H 'Content-Type: application/json' -d @test_postbodies/parallel_processing.json http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/mapsets/test_mapset/processing_parallel | jq -``` -Hereby, the parallel started process chains will be computed in mapsets with -the suffix `_parallel_{NUMBER}` (see the example process chains in -`test_postbodies/parallel_persistent_processing.json`). -So if you want to use a mapset in a later step, you have to pay attention to -the naming of the mapset. - -TODOs: alles diff --git a/docker/actinia-parallel-plugin-test/Dockerfile b/docker/actinia-parallel-plugin-test/Dockerfile index 3fbb270..1c86da7 100644 --- a/docker/actinia-parallel-plugin-test/Dockerfile +++ b/docker/actinia-parallel-plugin-test/Dockerfile @@ -9,16 +9,8 @@ ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia-parallel-plugin-test ENV DEFAULT_CONFIG_PATH /etc/default/actinia-parallel-plugin-test # install things only for tests -# RUN apk add postgresql -# RUN apk add docker openrc RUN apk add redis RUN pip3 install iniconfig colorlog pwgen -# RUN rc-update add docker boot -# RUN service docker start -# RUN docker pull kartoza/postgis - - -# COPY docker/actinia-parallel-plugin-test/start.sh /src/start.sh ENTRYPOINT ["/bin/sh"] CMD ["/src/start.sh"] @@ -28,7 +20,6 @@ RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_mic unzip nc_spm_08_micro.zip && \ rm -f nc_spm_08_micro.zip && \ mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08 -# RUN grass -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84 # copy needed files and configs for test COPY docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg /etc/default/actinia diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 7375a62..1a80e4e 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -75,7 +75,6 @@ class EnrichedRegeldateiModel(Schema): # 'description': 'List of processes to run', # 'items': EnrichedProcModel # } - # 'geodata_meta': "string" # TODO: GeodataResponseModel } # TODO add example # example = jobs_post_docs_request_example diff --git a/src/actinia_parallel_plugin/model/response_models.py b/src/actinia_parallel_plugin/model/response_models.py index 511b139..92c9621 100644 --- a/src/actinia_parallel_plugin/model/response_models.py +++ b/src/actinia_parallel_plugin/model/response_models.py @@ -49,36 +49,3 @@ class SimpleStatusCodeResponseModel(Schema): status=200, message="success" ) SimpleStatusCodeResponseModel.example = simpleResponseExample - - -class GeodataResponseModel(Schema): - """Model for object in enriched Regeldatei - - This object contains the metadata for input geodata from GNOS - """ - type = 'object' - properties = { - 'uuid': { - 'type': 'string', - 'description': 'The Geonetwork uuid.' - }, - 'bbox': { - 'type': 'array', - 'items': { - 'type': 'number' - }, - 'minItems': 4, - 'maxItems': 4, - 'description': 'The bounding box of the result.' - }, - 'crs': { - 'type': 'string', - 'description': 'The coordinate reference system of the result.' - }, - 'table': { - 'type': 'string', - 'description': ('The database connection string of the source ' + - 'of the result.') - } - } - required = ["uuid", "bbox"] From 3f7f2c7fafdae40125c0f08217934cd6b8f0d963 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 8 Jun 2022 08:56:15 +0200 Subject: [PATCH 30/47] CT review 2 --- .../api/parallel_ephemeral_processing.py | 2 +- src/actinia_parallel_plugin/core/parallel_processing_job.py | 2 +- src/actinia_parallel_plugin/endpoints.py | 2 +- src/actinia_parallel_plugin/model/response_models.py | 2 +- src/actinia_parallel_plugin/wsgi.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index 35d7c43..fcff4e5 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-present mundialis GmbH & Co. KG +Copyright (c) 2022 mundialis GmbH & Co. KG This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index a80f32e..dac68c6 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-present mundialis GmbH & Co. KG +Copyright (c) 2022 mundialis GmbH & Co. KG This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index 1537da1..803a89a 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-present mundialis GmbH & Co. KG +Copyright (c) 2022 mundialis GmbH & Co. KG This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/actinia_parallel_plugin/model/response_models.py b/src/actinia_parallel_plugin/model/response_models.py index 92c9621..1b51837 100644 --- a/src/actinia_parallel_plugin/model/response_models.py +++ b/src/actinia_parallel_plugin/model/response_models.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-present mundialis GmbH & Co. KG +Copyright (c) 2022 mundialis GmbH & Co. KG This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/actinia_parallel_plugin/wsgi.py b/src/actinia_parallel_plugin/wsgi.py index 1f60492..2f4358b 100644 --- a/src/actinia_parallel_plugin/wsgi.py +++ b/src/actinia_parallel_plugin/wsgi.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-present mundialis GmbH & Co. KG +Copyright (c) 2022 mundialis GmbH & Co. KG This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ """ __license__ = "GPLv3" -__author__ = "Carmen Tawalika, Anika Weinmann" +__author__ = "Anika Weinmann" __copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" From 93e817aa674bc915ee1afed944c0d7af480e7c06 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 8 Jun 2022 09:30:51 +0200 Subject: [PATCH 31/47] test workflow --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c5f20f..9a893dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,8 +27,7 @@ jobs: run: docker logs docker_postgis-test_1 - name: Run integration test - run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_unittests.sh - # run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh + run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh - name: Stop containers run: docker-compose -f "docker/docker-compose-test.yml" down From 80a3e352e32a52141c610d5674c4332a60b21031 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 29 Jun 2022 09:09:15 +0200 Subject: [PATCH 32/47] first cleanup --- .../batchjob_post_response_example.json | 3 - .../jobs_get_docs_response_example.json | 3 - src/actinia_parallel_plugin/apidocs/jobs.py | 14 ---- src/actinia_parallel_plugin/core/batches.py | 6 -- src/actinia_parallel_plugin/core/jobs.py | 68 ------------------- src/actinia_parallel_plugin/core/jobtable.py | 9 --- src/actinia_parallel_plugin/model/batch.py | 40 ++--------- .../model/jobtabelle.py | 28 ++++---- .../model/response_models.py | 51 -------------- tests/unittests/test_batches.py | 12 ---- 10 files changed, 20 insertions(+), 214 deletions(-) delete mode 100644 src/actinia_parallel_plugin/model/response_models.py diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index bf30167..c93ea61 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -5,8 +5,6 @@ "None", "None" ], - "actinia_core_platform": null, - "actinia_core_platform_name": null, "actinia_core_response": { "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18": { "accept_datetime": "2022-05-24 18:22:42.343953", @@ -34,7 +32,6 @@ "user_id": "actinia-gdi" } }, - "actinia_core_url": null, "actinia_gdi_batchid": 5, "batch_description": { "jobs": [ diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index 1aa4887..e99703c 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -1,7 +1,5 @@ { "actinia_core_jobid": "resource_id-71264d56-4183-4d68-8544-5425254f5def", - "actinia_core_platform": null, - "actinia_core_platform_name": null, "actinia_core_response": { "accept_datetime": "2022-05-24 18:22:12.481060", "accept_timestamp": 1653416532.4810588, @@ -113,7 +111,6 @@ }, "user_id": "actinia-gdi" }, - "actinia_core_url": null, "batch_description": { "jobs": [ { diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 1a80e4e..7eba62f 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -131,20 +131,6 @@ class ProcessesJobResponseModel(Schema): 'type': 'string', 'description': 'The actinia-core resource ID for the job' }, - 'actinia_core_platform': { - 'type': 'string', - 'description': 'The actinia-core platform, either "openshift" or ' - '"vm"' - }, - 'actinia_core_platform_name': { - 'type': 'string', - 'description': 'The actinia-core platform name' - }, - 'actinia_core_url': { - 'type': 'string', - 'description': 'The actinia-core IP or URL where actinia-core is ' - 'processing the job' - }, 'creation_uuid': { 'type': 'string', 'description': 'A unique id for the job at creation time before ' diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 7d64945..a0cfe76 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -242,10 +242,6 @@ def createBatchResponseDict(jobs_list): responseDict = { "actinia_gdi_batchid": batch_id, "actinia_core_jobid": resource_ids, - # some parameters will be the same in all jobs: - "actinia_core_platform": jobs[0]["actinia_core_platform"], - "actinia_core_platform_name": jobs[0]["actinia_core_platform_name"], - "actinia_core_url": jobs[0]["actinia_core_url"], "summary": summary_dict, "actinia_core_response": responses, "creation_uuids": uuids, @@ -309,8 +305,6 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, # "process": job["process"], # "process_chain": process_chain, # "jobid": job["idpk_jobs"], - # # "actinia_core_platform": job["actinia_core_platform"], - # # "actinia_core_url": job["actinia_core_url"] # } mapset_name_parallel = mapset_name if mapset_suffix != "" and mapset_name is not None: diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index fbb7978..d91c1a7 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -34,9 +34,6 @@ def insertJob(jsonDict, process, process_chain): """ function to prepare and call InsertNewJob from regeldatei""" - # actinia_core_url = None - # actinia_core_platform = None - # actinia_core_platform_name = None try: process_chain_struct = process_chain.to_struct() @@ -45,45 +42,10 @@ def insertJob(jsonDict, process, process_chain): log.error(e) return None - # vm_procs = PROCESSING.actinia_vm_processes.replace( - # ' ', '').split(',') - # - # # set default actinia connection parameter - # if (process in vm_procs): - # actinia_core_url = None - # actinia_core_platform = 'vm' - # else: - # actinia_core_url = ACTINIACORE.url - # actinia_core_platform = 'openshift' - # - # # overwrite actinia connection parameter if set in rulefile - # if (regeldatei.processing_platform): - # if (regeldatei.processing_platform.lower() == 'vm'): - # actinia_core_url = None - # actinia_core_platform = 'vm' - # elif (regeldatei.processing_platform.lower() == 'openshift'): - # actinia_core_url = ACTINIACORE.url - # actinia_core_platform = 'openshift' - # - # # overwrite actinia connection parameter if set in rulefile - # if (regeldatei.processing_host): - # actinia_core_url = regeldatei.processing_host - # if not actinia_core_url.startswith('http'): - # actinia_core_url = ACTINIACORE_VM.scheme + '://' + \ - # actinia_core_url - # if len(actinia_core_url.split(':')) == 2: - # actinia_core_url += ':' + ACTINIACORE_VM.port - # - # if (regeldatei.processing_platform_name): - # actinia_core_platform_name = regeldatei.processing_platform_name - job = insertNewJob( jsonDict, process_chain_struct, process, - # actinia_core_url, - # actinia_core_platform, - # actinia_core_platform_name ) return job @@ -128,34 +90,4 @@ def updateJob(resource_id, actinia_resp, jobid): resourceId=resource_id ) - # # TODO: for now if multiple records need to be updated (eg. for PT), this - # # can be told by specifying multiple uuids comma-separated in the - # # "feature_uuid" field of the rulefile. This might change later... - # if status == 'finished': - # try: - # gnosUuid = record['job_description']['feature_uuid'] - # utcnow = record['time_ended'] - # except Exception: - # log.warning('Feature has no uuid or time_ended') - # gnosUuid = None - # utcnow = None - # try: - # uuids = gnosUuid.split(',') - # for uuid in uuids: - # update(uuid, utcnow) - # except Exception: - # log.warning('Could not update geonetwork record') - - # # shutdown VM if process was calculated on VM - # terminate_status = ['finished', 'error', 'terminated'] - # processing_platform = record['actinia_core_platform'] - # process = record['process'] - # - # if (status in terminate_status - # and processing_platform is not None - # and processing_platform.lower() == 'vm' - # and 'processing_host' not in record['rule_configuration']): - # - # record = destroyVM(process, jobid) - return record diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index a99bdc4..084dd97 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -235,18 +235,12 @@ def insertNewJob( rule_configuration, job_description, process, - # feature_type, - actinia_core_url=None, - actinia_core_platform=None, - actinia_core_platform_name=None ): """Insert new job into jobtabelle. Args: rule_configuration (dict): original regeldatei job_description (TODO): enriched regeldatei with geometadata - actinia_core_url (string): url where processing will run - actinia_core_platform (string): platform where processing will run Returns: record (dict): the new record @@ -262,9 +256,6 @@ def insertNewJob( 'process': process, 'status': 'PREPARING', 'time_created': utcnow, - 'actinia_core_url': actinia_core_url, - 'actinia_core_platform': actinia_core_platform, - 'actinia_core_platform_name': actinia_core_platform_name, 'creation_uuid': creation_uuid } if "batch_id" in rule_configuration.keys(): diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 2ea8e02..9753dc7 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -141,26 +141,12 @@ class BatchProcessChainModel(Schema): """ type = 'object' properties = { - 'processing_host': { - 'type': 'string', - 'description': 'The actinia-core IP or URL in case the platform ' - 'is not OpenShift and no new VM should be created ' - 'by actinia-gdi' - }, - 'processing_platform': { - 'type': 'string', - 'description': 'The actinia-core platform, either "openshift" or ' - '"vm". If platform is "vm" and no actinia_core_url ' - 'is given, actinia-gdi will create a new VM.' - }, - 'processing_platform_name': { - 'type': 'string', - 'description': 'The actinia-core platform name. Only used to ' - 'match a job to a VM if VM not started by ' - 'actinia-gdi. Ideally it would contain the job ' - 'type (actinia-core-pt or actinia-core-oc) and ' - 'a unique ID.' - }, + # 'processing_host': { + # 'type': 'string', + # 'description': 'The actinia-core IP or URL in case the platform ' + # 'is not OpenShift and no new VM should be created ' + # 'by actinia-gdi' + # }, 'jobs': { 'type': 'array', 'items': ProcessChainModel, @@ -183,25 +169,11 @@ class BatchJobResponseModel(Schema): 'jobs'), 'items': {'type': 'string'} }, - 'actinia_core_platform': { - 'type': 'string', - 'description': ('The actinia-core platform, either "openshift"' - ' or "vm"') - }, - 'actinia_core_platform_name': { - 'type': 'string', - 'description': 'The actinia-core platform name' - }, 'actinia_core_response': { 'type': 'array', 'description': 'The responses of actinia-core for individual jobs', 'items': {'type': 'object'} }, - 'actinia_core_url': { - 'type': 'string', - 'description': ('The actinia-core IP or URL where actinia-core ' - 'is processing the batchjob') - }, 'actinia_gdi_batchid': { 'type': 'integer', 'description': 'The batch ID' diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 8edbf89..b632709 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -61,29 +61,29 @@ class Meta: class Job(BaseModel): """Model for jobtable in database """ - idpk_jobs = AutoField() - process = CharField(null=True) - # feature_type = CharField(null=True) - rule_configuration = BinaryJSONField(null=True) - job_description = BinaryJSONField(null=True) + # behalten time_created = DateTimeField(null=True) time_started = DateTimeField(null=True) time_estimated = DateTimeField(null=True) time_ended = DateTimeField(null=True) - # metadata = CharField(null=True) status = CharField(null=True) - actinia_core_response = BinaryJSONField(null=True) - actinia_core_jobid = CharField(null=True) - actinia_core_url = CharField(null=True) - actinia_core_platform = CharField(null=True) - actinia_core_platform_name = CharField(null=True) - terraformer_id = IntegerField(null=True) - terraformer_response = BinaryJSONField(null=True) + + actinia_core_response = BinaryJSONField(null=True) # resource_response + idpk_jobs = AutoField() # id + actinia_core_jobid = CharField(null=True) # resource_id + + # benötigt? creation_uuid = CharField(null=True) + process = CharField(null=True) message = CharField(null=True) + + # weg + rule_configuration = BinaryJSONField(null=True) + job_description = BinaryJSONField(null=True) + # add a potential parent_job batch_id = IntegerField(null=True) - processing_block = IntegerField(null=True) + processing_block = IntegerField(null=True) # batch_processing_block batch_description = BinaryJSONField(null=True) class Meta: diff --git a/src/actinia_parallel_plugin/model/response_models.py b/src/actinia_parallel_plugin/model/response_models.py deleted file mode 100644 index 1b51837..0000000 --- a/src/actinia_parallel_plugin/model/response_models.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2022 mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Response models -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - - -from flask_restful_swagger_2 import Schema - - -class SimpleStatusCodeResponseModel(Schema): - """Simple response schema to inform about status.""" - - type = "object" - properties = { - "status": { - "type": "number", - "description": "The status code of the request.", - }, - "message": { - "type": "string", - "description": "A short message to describes the status", - }, - } - required = ["status", "message"] - - -simpleResponseExample = SimpleStatusCodeResponseModel( - status=200, message="success" -) -SimpleStatusCodeResponseModel.example = simpleResponseExample diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 82f1237..49c192e 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -72,9 +72,6 @@ "status": "PREPARING", "actinia_core_response": None, "actinia_core_jobid": None, - "actinia_core_url": None, - "actinia_core_platform": None, - "actinia_core_platform_name": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", @@ -139,9 +136,6 @@ "status": "PREPARING", "actinia_core_response": None, "actinia_core_jobid": None, - "actinia_core_url": None, - "actinia_core_platform": None, - "actinia_core_platform_name": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", @@ -230,9 +224,6 @@ ], }, "actinia_core_jobid": resource_id2, - "actinia_core_url": None, - "actinia_core_platform": None, - "actinia_core_platform_name": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", @@ -321,9 +312,6 @@ ], }, "actinia_core_jobid": resource_id1, - "actinia_core_url": None, - "actinia_core_platform": None, - "actinia_core_platform_name": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", From 114dee373d1b8d5512b5b4bb28f1c14774080224 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 29 Jun 2022 10:15:50 +0200 Subject: [PATCH 33/47] remove job_description --- .../jobs_get_docs_response_example.json | 301 ------------------ src/actinia_parallel_plugin/apidocs/jobs.py | 41 --- src/actinia_parallel_plugin/core/jobs.py | 1 - src/actinia_parallel_plugin/core/jobtable.py | 3 - src/actinia_parallel_plugin/model/batch.py | 4 +- .../model/jobtabelle.py | 8 +- tests/unittests/test_batches.py | 66 ---- 7 files changed, 6 insertions(+), 418 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index e99703c..3fd098a 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -225,307 +225,6 @@ "batch_id": 3, "creation_uuid": "49cdd55a-332b-4f66-ad40-c0b3e576f824", "idpk_jobs": 11, - "job_description": { - "batch_description": { - "jobs": [ - { - "list": [ - { - "id": "g_region_nonparallel_block1", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region", - "outputs": [] - }, - { - "id": "r_mapcalc_0_nonparallel_block1", - "inputs": [ - { - "param": "expression", - "value": "baum = elevation@PERMANENT * 2" - } - ], - "module": "r.mapcalc", - "outputs": [] - } - ], - "parallel": "false", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_1_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region", - "outputs": [] - }, - { - "id": "r_info_1_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elevation@PERMANENT" - } - ], - "module": "r.info", - "outputs": [] - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region", - "outputs": [] - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "outputs": [], - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_nonparallel_block3", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region", - "outputs": [] - } - ], - "parallel": "false", - "version": "1" - } - ] - }, - "batch_id": 3, - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region", - "outputs": [] - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "outputs": [], - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "processing_block": 2, - "version": "1" - }, - "message": null, - "process": "ephemeral", - "processing_block": 2, - "rule_configuration": { - "batch_description": { - "jobs": [ - { - "list": [ - { - "id": "g_region_nonparallel_block1", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_mapcalc_0_nonparallel_block1", - "inputs": [ - { - "param": "expression", - "value": "baum = elevation@PERMANENT * 2" - } - ], - "module": "r.mapcalc" - } - ], - "parallel": "false", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_1_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_info_1_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elevation@PERMANENT" - } - ], - "module": "r.info" - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "import_descr": { - "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", - "type": "raster" - }, - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region" - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_nonparallel_block3", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - } - ], - "parallel": "false", - "version": "1" - } - ] - }, - "batch_id": 3, - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region", - "outputs": [] - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "outputs": [], - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "parallel": "true", - "processing_block": 2, - "version": "1" - }, "status": "SUCCESS", "terraformer_id": null, "terraformer_response": null, diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 7eba62f..b0d8f94 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -41,46 +41,6 @@ jobs_get_docs_response_example = json.load(jsonfile) -class EnrichedRegeldateiModel(Schema): - """Request schema for creating a job""" - # TODO check if this is correct - type = 'object' - properties = { - 'rule_area_id': { - 'type': 'integer', - 'description': 'Identifier of area where Regeldatei is valid' - }, - 'rule_area': { - 'type': 'string', - 'description': 'Name of area where Regeldatei is valid' - }, - 'feature_uuid': { - 'type': 'string', - 'description': 'Geonetwork UUID of feature type to run job with' - }, - 'processing_platform': { - 'type': 'string', - 'description': 'TODO' - }, - 'processing_platform_name': { - 'type': 'string', - 'description': 'TODO (and a unique ID.)' - }, - 'processing_host': { - 'type': 'string', - 'description': 'TODO (The actinia-core IP or URL)' - } - # 'procs': { - # 'type': 'array', - # 'description': 'List of processes to run', - # 'items': EnrichedProcModel - # } - } - # TODO add example - # example = jobs_post_docs_request_example - required = ["feature_source"] - - class ProcessesJobResponseModel(Schema): """Response schema for creating a job""" type = 'object' @@ -95,7 +55,6 @@ class ProcessesJobResponseModel(Schema): 'or potentialtrenches' }, 'rule_configuration': RegeldateiModel, - 'job_description': EnrichedRegeldateiModel, 'time_created': { 'type': 'string', 'description': 'Timestamp when job was created' diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index d91c1a7..b873766 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -44,7 +44,6 @@ def insertJob(jsonDict, process, process_chain): job = insertNewJob( jsonDict, - process_chain_struct, process, ) return job diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 084dd97..72d6fab 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -233,14 +233,12 @@ def getJobByResource(key, val): def insertNewJob( rule_configuration, - job_description, process, ): """Insert new job into jobtabelle. Args: rule_configuration (dict): original regeldatei - job_description (TODO): enriched regeldatei with geometadata Returns: record (dict): the new record @@ -252,7 +250,6 @@ def insertNewJob( job_kwargs = { 'rule_configuration': rule_configuration, - 'job_description': job_description, 'process': process, 'status': 'PREPARING', 'time_created': utcnow, diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 9753dc7..0bd260d 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -144,8 +144,8 @@ class BatchProcessChainModel(Schema): # 'processing_host': { # 'type': 'string', # 'description': 'The actinia-core IP or URL in case the platform ' - # 'is not OpenShift and no new VM should be created ' - # 'by actinia-gdi' + # 'is not OpenShift and no new VM should be created' + # ' by actinia-gdi' # }, 'jobs': { 'type': 'array', diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index b632709..00ce06f 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -67,24 +67,24 @@ class Job(BaseModel): time_estimated = DateTimeField(null=True) time_ended = DateTimeField(null=True) status = CharField(null=True) + creation_uuid = CharField(null=True) actinia_core_response = BinaryJSONField(null=True) # resource_response idpk_jobs = AutoField() # id actinia_core_jobid = CharField(null=True) # resource_id # benötigt? - creation_uuid = CharField(null=True) process = CharField(null=True) message = CharField(null=True) # weg - rule_configuration = BinaryJSONField(null=True) - job_description = BinaryJSONField(null=True) + rule_configuration = BinaryJSONField(null=True) # WIRD VERWENDET!!! vielleicht eher batch_description löschen? + # job_description = BinaryJSONField(null=True) + batch_description = BinaryJSONField(null=True) # weg als eintrag # add a potential parent_job batch_id = IntegerField(null=True) processing_block = IntegerField(null=True) # batch_processing_block - batch_description = BinaryJSONField(null=True) class Meta: table_name = JOBTABLE.table diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 49c192e..8c8b2ef 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -51,20 +51,6 @@ ] }, }, - "job_description": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 930771), "time_started": None, "time_estimated": None, @@ -105,30 +91,6 @@ ] }, }, - "job_description": { - "list": [ - { - "id": "g_region_nonparallel_block3", - "flags": "p", - "inputs": [ - {"param": "raster", "value": "elevation@PERMANENT"} - ], - "module": "g.region", - "outputs": [], - } - ], - "version": "1", - "batch_id": 2, - "processing_block": 3, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 940472), "time_started": None, "time_estimated": None, @@ -169,20 +131,6 @@ ] }, }, - "job_description": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "processing_block": 1, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 906827), "time_started": None, "time_estimated": None, @@ -257,20 +205,6 @@ ] }, }, - "job_description": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 920875), "time_started": None, "time_estimated": None, From 894458a3308e000ed2724f1ea08a5b6ae9c6bc64 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 29 Jun 2022 13:57:44 +0200 Subject: [PATCH 34/47] renaming processing_block to batch_processing_block --- src/actinia_parallel_plugin/core/batches.py | 16 ++++++++-------- .../core/ephemeral_processing.py | 8 ++++---- src/actinia_parallel_plugin/core/jobtable.py | 2 +- .../model/batch_process_chain.py | 6 +++--- src/actinia_parallel_plugin/model/jobtabelle.py | 2 +- tests/unittests/test_batches.py | 16 ++++++++-------- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index a0cfe76..3058f0e 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -74,7 +74,7 @@ def assignProcessingBlocks(jsonDict): prev_parallel = parallel_jobs_corrected[idx-1] if idx > 0 and (parallel is False or prev_parallel is False): block_num += 1 - job["processing_block"] = block_num + job["batch_processing_block"] = block_num result_jobs.append(job) return result_jobs @@ -118,7 +118,7 @@ def checkProcessingBlockFinished(jobs, block): an input list of jobs (db entries) """ status_list = [job["status"] for job - in jobs if job["processing_block"] == block] + in jobs if job["batch_processing_block"] == block] finished = all(status == "SUCCESS" for status in status_list) return finished @@ -191,7 +191,7 @@ def createBatchResponseDict(jobs_list): # status.append(str(job["status"])) uuids.append(job["creation_uuid"]) job_ids.append(str(job["idpk_jobs"])) - blocks.append(job["processing_block"]) + blocks.append(job["batch_processing_block"]) # determine an overall batch status overall_status_list = [job["status"] for job in jobs_status] @@ -214,10 +214,10 @@ def createBatchResponseDict(jobs_list): batch_status = "RUNNING" # create block-wise statistics - processing_blocks = [] + batch_processing_blocks = [] for block in sorted(set(blocks)): status_list = [job["status"] for job in jobs if - job["processing_block"] == block] + job["batch_processing_block"] == block] status_dict = _count_status_from_list(status_list) if len(status_list) > 1: parallel = len(status_list) @@ -229,13 +229,13 @@ def createBatchResponseDict(jobs_list): "parallel": parallel } block_stats = {**block_info, **status_dict} - processing_blocks.append(block_stats) + batch_processing_blocks.append(block_stats) # create summary statistics summary_dict = { "total": len(job_ids), "status": _count_status_from_list(overall_status_list), - "blocks": processing_blocks + "blocks": batch_processing_blocks } # create overall response dict @@ -291,7 +291,7 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, jobs (db entries) """ jobs_to_start = [ - job for job in jobs if job["processing_block"] == block] + job for job in jobs if job["batch_processing_block"] == block] jobs_responses = [] mapset_suffix = "" if len(jobs_to_start) > 1: diff --git a/src/actinia_parallel_plugin/core/ephemeral_processing.py b/src/actinia_parallel_plugin/core/ephemeral_processing.py index 954fbf7..bf0672b 100644 --- a/src/actinia_parallel_plugin/core/ephemeral_processing.py +++ b/src/actinia_parallel_plugin/core/ephemeral_processing.py @@ -40,12 +40,12 @@ class ParallelEphemeralProcessing(EphemeralProcessing): - def __init__(self, rdc, batch_id, processing_block, jobid, + def __init__(self, rdc, batch_id, batch_processing_block, jobid, user, request_url, post_url, endpoint, method, path, base_status_url): super(ParallelEphemeralProcessing, self).__init__(rdc) self.batch_id = batch_id - self.processing_block = processing_block + self.batch_processing_block = batch_processing_block self.jobid = jobid self.post_url = post_url self.user = user @@ -78,8 +78,8 @@ def _update_and_check_batch_jobs(self): "ephemeral" ) all_blocks = [ - job["processing_block"] for job in jobs_from_batch] - block = int(self.processing_block) + job["batch_processing_block"] for job in jobs_from_batch] + block = int(self.batch_processing_block) block_done = checkProcessingBlockFinished( jobs_from_batch, block) if block_done is True and block < max(all_blocks): diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 72d6fab..be29c53 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -257,7 +257,7 @@ def insertNewJob( } if "batch_id" in rule_configuration.keys(): # then it's a batch job - job_kwargs["processing_block"] = rule_configuration["processing_block"] + job_kwargs["batch_processing_block"] = rule_configuration["batch_processing_block"] job_kwargs["batch_id"] = rule_configuration["batch_id"] job_kwargs["batch_description"] = rule_configuration[ "batch_description"] diff --git a/src/actinia_parallel_plugin/model/batch_process_chain.py b/src/actinia_parallel_plugin/model/batch_process_chain.py index f481a49..7a4b056 100644 --- a/src/actinia_parallel_plugin/model/batch_process_chain.py +++ b/src/actinia_parallel_plugin/model/batch_process_chain.py @@ -88,7 +88,7 @@ class Job(models.Base): parallel = fields.StringField(required=True) # bool list = fields.ListField([Module], required=True) # array of objects # the block and batch id is not in the json but is filled later - processing_block = fields.IntField() + batch_processing_block = fields.IntField() batch_id = fields.IntField() @@ -121,9 +121,9 @@ class SingleJob(models.Base): processing_platform = fields.StringField() # string processing_platform_name = fields.StringField() # string processing_host = fields.StringField() # string - # processing_block, batch_description and + # batch_processing_block, batch_description and # batch id are not in the json but is filled later - processing_block = fields.IntField() + batch_processing_block = fields.IntField() batch_id = fields.IntField() # batch_description holds the entire batch processing chain batch_description = fields.EmbeddedField(BatchProcessChain) diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 00ce06f..397d578 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -84,7 +84,7 @@ class Job(BaseModel): # add a potential parent_job batch_id = IntegerField(null=True) - processing_block = IntegerField(null=True) # batch_processing_block + batch_processing_block = IntegerField(null=True) # batch_batch_processing_block class Meta: table_name = JOBTABLE.table diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 8c8b2ef..e9d58e9 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -41,7 +41,7 @@ "version": "1", "batch_id": 2, "parallel": "true", - "processing_block": 2, + "batch_processing_block": 2, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -63,7 +63,7 @@ "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", "message": None, "batch_id": 2, - "processing_block": 2, + "batch_processing_block": 2, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -81,7 +81,7 @@ "version": "1", "batch_id": 2, "parallel": "false", - "processing_block": 3, + "batch_processing_block": 3, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -103,7 +103,7 @@ "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", "message": None, "batch_id": 2, - "processing_block": 3, + "batch_processing_block": 3, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -121,7 +121,7 @@ "version": "1", "batch_id": 2, "parallel": "false", - "processing_block": 1, + "batch_processing_block": 1, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -177,7 +177,7 @@ "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", "message": None, "batch_id": 2, - "processing_block": 1, + "batch_processing_block": 1, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -195,7 +195,7 @@ "version": "1", "batch_id": 2, "parallel": "true", - "processing_block": 2, + "batch_processing_block": 2, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, @@ -251,7 +251,7 @@ "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", "message": None, "batch_id": 2, - "processing_block": 2, + "batch_processing_block": 2, "batch_description": { "jobs": [ {"list": [{"..."}], "version": "1", "parallel": "false"}, From eaec58c87a8b0e4ad035e3136eb8bb357d3a2e19 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 29 Jun 2022 15:09:32 +0200 Subject: [PATCH 35/47] renaming to resource_id --- .../examples/batchjob_post_response_example.json | 10 +++++----- .../examples/jobs_get_docs_response_example.json | 2 +- src/actinia_parallel_plugin/apidocs/jobs.py | 4 ++-- src/actinia_parallel_plugin/core/batches.py | 6 +++--- src/actinia_parallel_plugin/core/jobs.py | 2 +- src/actinia_parallel_plugin/core/jobtable.py | 6 +++--- src/actinia_parallel_plugin/model/batch.py | 4 ++-- src/actinia_parallel_plugin/model/jobtabelle.py | 5 ++--- src/actinia_parallel_plugin/resources/config.py | 2 +- tests/unittests/test_batches.py | 8 ++++---- 10 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index c93ea61..44b6990 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -1,5 +1,5 @@ { - "actinia_core_jobid": [ + "resource_id": [ "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", "None", "None", @@ -158,22 +158,22 @@ ], "jobs_status": [ { - "actinia_core_jobid": "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", + "resource_id": "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", "idpk_jobs": 17, "status": "PENDING" }, { - "actinia_core_jobid": "None", + "resource_id": "None", "idpk_jobs": 18, "status": "PREPARING" }, { - "actinia_core_jobid": "None", + "resource_id": "None", "idpk_jobs": 19, "status": "PREPARING" }, { - "actinia_core_jobid": "None", + "resource_id": "None", "idpk_jobs": 20, "status": "PREPARING" } diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index 3fd098a..a4b1bb4 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -1,5 +1,5 @@ { - "actinia_core_jobid": "resource_id-71264d56-4183-4d68-8544-5425254f5def", + "resource_id": "resource_id-71264d56-4183-4d68-8544-5425254f5def", "actinia_core_response": { "accept_datetime": "2022-05-24 18:22:12.481060", "accept_timestamp": 1653416532.4810588, diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index b0d8f94..0c30e63 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -86,9 +86,9 @@ class ProcessesJobResponseModel(Schema): 'type': 'object', 'description': 'The Response of actinia-core at creation time' }, - 'actinia_core_jobid': { + 'resource_id': { 'type': 'string', - 'description': 'The actinia-core resource ID for the job' + 'description': 'The resource ID for the job' }, 'creation_uuid': { 'type': 'string', diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 3058f0e..c9c89a1 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -177,14 +177,14 @@ def createBatchResponseDict(jobs_list): job_ids = [] blocks = [] for job in jobs: - resource_id = job["actinia_core_jobid"] + resource_id = job["resource_id"] # this way we also have "None" if no resource_id is given yet: resource_ids.append(str(resource_id)) if resource_id is not None: responses[resource_id] = job["actinia_core_response"] job_status = { "idpk_jobs": job["idpk_jobs"], - "actinia_core_jobid": str(resource_id), + "resource_id": str(resource_id), "status": str(job["status"]) } jobs_status.append(job_status) @@ -241,7 +241,7 @@ def createBatchResponseDict(jobs_list): # create overall response dict responseDict = { "actinia_gdi_batchid": batch_id, - "actinia_core_jobid": resource_ids, + "resource_id": resource_ids, "summary": summary_dict, "actinia_core_response": responses, "creation_uuids": uuids, diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index b873766..7b30709 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -79,7 +79,7 @@ def updateJob(resource_id, actinia_resp, jobid): """ status = actinia_resp["status"] - # record = getJobByResource("actinia_core_jobid", resource_id) + # record = getJobByResource("resource_id", resource_id) # follow-up actinia update, therefore without resourceId record = updateJobByID( diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index be29c53..25eb187 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -336,7 +336,7 @@ def updateJobByID( updatekwargs = { 'status': status, 'actinia_core_response': resp, - 'actinia_core_jobid': resourceId + 'resource_id': resourceId } if message is not None: updatekwargs['message'] = message @@ -362,7 +362,7 @@ def updateJobByID( if message is not None: updatekwargs['message'] = message if resourceId is not None: - updatekwargs['actinia_core_jobid'] = resourceId + updatekwargs['resource_id'] = resourceId # TODO: check if time_estimated can be set # time_estimated= @@ -381,7 +381,7 @@ def updateJobByID( if message is not None: updatekwargs['message'] = message if resourceId is not None: - updatekwargs['actinia_core_jobid'] = resourceId + updatekwargs['resource_id'] = resourceId query = Job.update(**updatekwargs).where( getattr(Job, JOBTABLE.id_field) == jobid diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 0bd260d..bd27620 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -163,9 +163,9 @@ class BatchJobResponseModel(Schema): """ type = 'object' properties = { - 'actinia_core_jobid': { + 'resource_id': { 'type': 'array', - 'description': ('The actinia-core resource IDs for all individual ' + 'description': ('The resource IDs for all individual ' 'jobs'), 'items': {'type': 'string'} }, diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 397d578..747cf44 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -69,9 +69,9 @@ class Job(BaseModel): status = CharField(null=True) creation_uuid = CharField(null=True) - actinia_core_response = BinaryJSONField(null=True) # resource_response + actinia_core_response = BinaryJSONField(null=True) # resource_response !!! idpk_jobs = AutoField() # id - actinia_core_jobid = CharField(null=True) # resource_id + resource_id = CharField(null=True) # benötigt? process = CharField(null=True) @@ -79,7 +79,6 @@ class Job(BaseModel): # weg rule_configuration = BinaryJSONField(null=True) # WIRD VERWENDET!!! vielleicht eher batch_description löschen? - # job_description = BinaryJSONField(null=True) batch_description = BinaryJSONField(null=True) # weg als eintrag # add a potential parent_job diff --git a/src/actinia_parallel_plugin/resources/config.py b/src/actinia_parallel_plugin/resources/config.py index f098d98..01b74d0 100644 --- a/src/actinia_parallel_plugin/resources/config.py +++ b/src/actinia_parallel_plugin/resources/config.py @@ -56,7 +56,7 @@ class JOBTABLE: table = 'tab_jobs' id_field = 'idpk_jobs' batch_id_field = "batch_id" - resource_id_field = "actinia_core_jobid" + resource_id_field = "resource_id" class LOGCONFIG: diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index e9d58e9..1be8282 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -57,7 +57,7 @@ "time_ended": None, "status": "PREPARING", "actinia_core_response": None, - "actinia_core_jobid": None, + "resource_id": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", @@ -97,7 +97,7 @@ "time_ended": None, "status": "PREPARING", "actinia_core_response": None, - "actinia_core_jobid": None, + "resource_id": None, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", @@ -171,7 +171,7 @@ } ], }, - "actinia_core_jobid": resource_id2, + "resource_id": resource_id2, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", @@ -245,7 +245,7 @@ } ], }, - "actinia_core_jobid": resource_id1, + "resource_id": resource_id1, "terraformer_id": None, "terraformer_response": None, "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", From 1b5df75d7697ce9cbb1801ddd777a6ffb8f65852 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 29 Jun 2022 16:40:00 +0200 Subject: [PATCH 36/47] remove message and terraformer_.. --- .../jobs_get_docs_response_example.json | 2 -- src/actinia_parallel_plugin/apidocs/jobs.py | 8 ------ src/actinia_parallel_plugin/core/jobtable.py | 26 ++----------------- .../model/jobtabelle.py | 4 ++- tests/unittests/test_batches.py | 8 ------ 5 files changed, 5 insertions(+), 43 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index a4b1bb4..f0aa6fd 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -226,8 +226,6 @@ "creation_uuid": "49cdd55a-332b-4f66-ad40-c0b3e576f824", "idpk_jobs": 11, "status": "SUCCESS", - "terraformer_id": null, - "terraformer_response": null, "time_created": "Tue, 24 May 2022 18:22:09 GMT", "time_ended": "Tue, 24 May 2022 18:22:12 GMT", "time_estimated": null, diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 0c30e63..db15c7a 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -95,14 +95,6 @@ class ProcessesJobResponseModel(Schema): 'description': 'A unique id for the job at creation time before ' 'idpk_jobs is known. (More unique than creation ' 'timestamp)' - }, - 'terraformer_id': { - 'type': 'string', - 'description': 'The terraformer instances ID for the job' - }, - 'terraformer_response': { - 'type': 'string', - 'description': 'The Response/Status of terraformer' } } example = jobs_get_docs_response_example diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 25eb187..454b4ac 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -281,7 +281,7 @@ def insertNewJob( def updateJobByID( - jobid, status, resp, resourceId=None, message=None): + jobid, status, resp, resourceId=None): """ Method to update job in jobtabelle when processing status changed Args: @@ -289,19 +289,12 @@ def updateJobByID( status (string): actinia-core processing status resp (dict): actinia-core response resourceId (str): actinia-core resourceId - message (str): general message for the job Returns: updatedRecord (TODO): the updated record """ - # terraformer ["PENDING", "STARTING", "STARTED", "INSTALLING", "RUNNING", - # "ERROR", "TERMINATING", "TERMINATED"] - # terraformer ERROR leads to ERROR, else PREPARING or SUCCESS - # actinia-gdi ["PREPARING"] - # actinia-gdi ["PENDING", "RUNNING", "SUCCESS", "ERROR", "TERMINATED"] - # actinia-core [accepted, running, finished, error, terminated] - + # TODO do not rename the status (kleineschreiben!!!) if status == 'accepted': status = 'PENDING' elif status == 'running': @@ -321,13 +314,6 @@ def updateJobByID( dbStatus = record['status'] try: - # outcommented to see if more feasable if whole logs are passed - # if current_app.debug is False: - # smallRes = dict() - # smallRes['message'] = resp.get('message', None) - # smallRes['process_results'] = resp.get('process_results', None) - # resp = smallRes - if status == 'PENDING': if dbStatus == status: return record @@ -338,8 +324,6 @@ def updateJobByID( 'actinia_core_response': resp, 'resource_id': resourceId } - if message is not None: - updatekwargs['message'] = message query = Job.update(**updatekwargs).where( getattr(Job, JOBTABLE.id_field) == jobid @@ -350,8 +334,6 @@ def updateJobByID( if dbStatus == status: updatekwargs['actinia_core_response'] = resp - if message is not None: - updatekwargs['message'] = message else: log.debug("Update status to " + status + " for job with id " @@ -359,8 +341,6 @@ def updateJobByID( updatekwargs['status'] = status updatekwargs['actinia_core_response'] = resp updatekwargs['time_started'] = utcnow - if message is not None: - updatekwargs['message'] = message if resourceId is not None: updatekwargs['resource_id'] = resourceId # TODO: check if time_estimated can be set @@ -378,8 +358,6 @@ def updateJobByID( 'actinia_core_response': resp, 'time_ended': utcnow } - if message is not None: - updatekwargs['message'] = message if resourceId is not None: updatekwargs['resource_id'] = resourceId diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 747cf44..d9831b3 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -71,11 +71,13 @@ class Job(BaseModel): actinia_core_response = BinaryJSONField(null=True) # resource_response !!! idpk_jobs = AutoField() # id + # DETAIL: Key (relname, relnamespace)=(tab_jobs_id_job_seq, 16386) ' + # 'already exists. resource_id = CharField(null=True) # benötigt? process = CharField(null=True) - message = CharField(null=True) + # message = CharField(null=True) # weg rule_configuration = BinaryJSONField(null=True) # WIRD VERWENDET!!! vielleicht eher batch_description löschen? diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 1be8282..7b01070 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -58,8 +58,6 @@ "status": "PREPARING", "actinia_core_response": None, "resource_id": None, - "terraformer_id": None, - "terraformer_response": None, "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", "message": None, "batch_id": 2, @@ -98,8 +96,6 @@ "status": "PREPARING", "actinia_core_response": None, "resource_id": None, - "terraformer_id": None, - "terraformer_response": None, "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", "message": None, "batch_id": 2, @@ -172,8 +168,6 @@ ], }, "resource_id": resource_id2, - "terraformer_id": None, - "terraformer_response": None, "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", "message": None, "batch_id": 2, @@ -246,8 +240,6 @@ ], }, "resource_id": resource_id1, - "terraformer_id": None, - "terraformer_response": None, "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", "message": None, "batch_id": 2, From 88fa151eab694a46e08abb412a3ed0f3451cab71 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 08:43:35 +0200 Subject: [PATCH 37/47] rename idpk_jobs in id --- docker/actinia.cfg | 2 +- .../api/parallel_ephemeral_processing.py | 1 - .../examples/batchjob_post_response_example.json | 10 +++++----- .../examples/jobs_get_docs_response_example.json | 2 +- src/actinia_parallel_plugin/apidocs/jobs.py | 5 ++--- src/actinia_parallel_plugin/core/batches.py | 15 +++++---------- src/actinia_parallel_plugin/core/jobtable.py | 15 ++++----------- src/actinia_parallel_plugin/model/batch.py | 6 +++--- src/actinia_parallel_plugin/model/jobtabelle.py | 4 +--- src/actinia_parallel_plugin/resources/config.py | 2 +- tests/unittests/test_batches.py | 8 ++++---- 11 files changed, 27 insertions(+), 43 deletions(-) diff --git a/docker/actinia.cfg b/docker/actinia.cfg index 3fa6046..54d7055 100644 --- a/docker/actinia.cfg +++ b/docker/actinia.cfg @@ -34,4 +34,4 @@ database = gis user = actinia schema = actinia table = tab_jobs -id_field = idpk_jobs +id_field = id diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index fcff4e5..62a43e6 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -70,7 +70,6 @@ def __init__(self): self.batch_id = None @swagger.doc(batch.batchjobs_post_docs) - # def get(self): def post(self, location_name): """Persistent parallel processing.""" diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index 44b6990..ea581e5 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -150,7 +150,7 @@ "5a91e046-faec-414d-8013-771747196126", "ee72667f-248a-4200-8c2c-e6025fa80254" ], - "idpk_jobs": [ + "id": [ "17", "18", "19", @@ -159,22 +159,22 @@ "jobs_status": [ { "resource_id": "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18", - "idpk_jobs": 17, + "id": 17, "status": "PENDING" }, { "resource_id": "None", - "idpk_jobs": 18, + "id": 18, "status": "PREPARING" }, { "resource_id": "None", - "idpk_jobs": 19, + "id": 19, "status": "PREPARING" }, { "resource_id": "None", - "idpk_jobs": 20, + "id": 20, "status": "PREPARING" } ], diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index f0aa6fd..e944119 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -224,7 +224,7 @@ }, "batch_id": 3, "creation_uuid": "49cdd55a-332b-4f66-ad40-c0b3e576f824", - "idpk_jobs": 11, + "id": 11, "status": "SUCCESS", "time_created": "Tue, 24 May 2022 18:22:09 GMT", "time_ended": "Tue, 24 May 2022 18:22:12 GMT", diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index db15c7a..5bde0bb 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -36,7 +36,6 @@ rel_path = "../apidocs/examples/jobs_get_docs_response_example.json" abs_file_path = os.path.join(script_dir, rel_path) -print(abs_file_path) with open(abs_file_path) as jsonfile: jobs_get_docs_response_example = json.load(jsonfile) @@ -45,7 +44,7 @@ class ProcessesJobResponseModel(Schema): """Response schema for creating a job""" type = 'object' properties = { - 'idpk_jobs': { + 'id': { 'type': 'integer', 'description': 'The job ID' }, @@ -93,7 +92,7 @@ class ProcessesJobResponseModel(Schema): 'creation_uuid': { 'type': 'string', 'description': 'A unique id for the job at creation time before ' - 'idpk_jobs is known. (More unique than creation ' + 'id is known. (More unique than creation ' 'timestamp)' } } diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index c9c89a1..2c2b1c9 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -167,7 +167,7 @@ def createBatchResponseDict(jobs_list): return {} # sort the jobs according to their id - jobs = sorted(jobs_list, key=lambda d: d["idpk_jobs"]) + jobs = sorted(jobs_list, key=lambda d: d["id"]) process = jobs[0]["process"] batch_id = jobs[0]["batch_id"] resource_ids = [] @@ -183,14 +183,14 @@ def createBatchResponseDict(jobs_list): if resource_id is not None: responses[resource_id] = job["actinia_core_response"] job_status = { - "idpk_jobs": job["idpk_jobs"], + "id": job["id"], "resource_id": str(resource_id), "status": str(job["status"]) } jobs_status.append(job_status) # status.append(str(job["status"])) uuids.append(job["creation_uuid"]) - job_ids.append(str(job["idpk_jobs"])) + job_ids.append(str(job["id"])) blocks.append(job["batch_processing_block"]) # determine an overall batch status @@ -245,7 +245,7 @@ def createBatchResponseDict(jobs_list): "summary": summary_dict, "actinia_core_response": responses, "creation_uuids": uuids, - "idpk_jobs": job_ids, + "id": job_ids, "batch_description": jobs[0]["batch_description"], "process": process, "jobs_status": jobs_status, @@ -300,12 +300,7 @@ def startProcessingBlock(jobs, block, batch_id, location_name, mapset_name, process_chain = dict() process_chain["list"] = job["rule_configuration"]["list"] process_chain["version"] = job["rule_configuration"]["version"] - jobid = job["idpk_jobs"] - # start_kwargs = { - # "process": job["process"], - # "process_chain": process_chain, - # "jobid": job["idpk_jobs"], - # } + jobid = job["id"] mapset_name_parallel = mapset_name if mapset_suffix != "" and mapset_name is not None: mapset_name_parallel += f"{mapset_suffix}{num}" diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 454b4ac..09e4c3c 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -172,8 +172,6 @@ def getJobById(jobid): queryResult = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult) - # log.info("Information read from jobtable for job with id " - # + str(record['idpk_jobs']) + ".") err = None except Job.DoesNotExist: record = None @@ -220,8 +218,6 @@ def getJobByResource(key, val): queryResult = Job.select().where( getattr(Job, key) == val).get() record = model_to_dict(queryResult) - # log.info("Information read from jobtable for job with id " - # + str(record['idpk_jobs']) + ".") except Job.DoesNotExist: record = None @@ -273,7 +269,7 @@ def insertNewJob( record = model_to_dict(queryResult) - log.info("Created new job with id " + str(record['idpk_jobs']) + ".") + log.info("Created new job with id " + str(record['id']) + ".") jobdb.close() @@ -318,7 +314,7 @@ def updateJobByID( if dbStatus == status: return record log.debug("Update status to " + status + " for job with id " - + str(record['idpk_jobs']) + ".") + + str(record['id']) + ".") updatekwargs = { 'status': status, 'actinia_core_response': resp, @@ -337,7 +333,7 @@ def updateJobByID( else: log.debug("Update status to " + status + " for job with id " - + str(record['idpk_jobs']) + ".") + + str(record['id']) + ".") updatekwargs['status'] = status updatekwargs['actinia_core_response'] = resp updatekwargs['time_started'] = utcnow @@ -352,7 +348,7 @@ def updateJobByID( elif status in ['SUCCESS', 'ERROR', 'TERMINATED']: log.debug("Update status to " + status + " for job with id " - + str(record['idpk_jobs']) + ".") + + str(record['id']) + ".") updatekwargs = { 'status': status, 'actinia_core_response': resp, @@ -381,9 +377,6 @@ def updateJobByID( log.error(str(e)) return None - # log.debug("Updated status to " + status + " for job with id " - # + str(record['idpk_jobs']) + ".") - jobdb.close() return record diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index bd27620..21d0e6a 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -182,11 +182,11 @@ class BatchJobResponseModel(Schema): 'creation_uuid': { 'type': 'array', 'description': ('Unique ids for the individual jobs at creation ' - 'time before idpk_jobs is known. ' + 'time before id is known. ' '(More unique than creation timestamp)'), 'items': {'type': 'string'} }, - 'idpk_jobs': { + 'id': { 'type': 'array', 'description': 'The individual job IDs', 'items': {'type': 'integer'} @@ -216,7 +216,7 @@ class BatchJobResponseModel(Schema): 'type': 'string', 'description': 'The actinia-core resource ID for the job' }, - 'idpk_jobs': { + 'id': { 'type': 'integer', 'description': 'The job ID' }, diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index d9831b3..9a2546e 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -70,9 +70,7 @@ class Job(BaseModel): creation_uuid = CharField(null=True) actinia_core_response = BinaryJSONField(null=True) # resource_response !!! - idpk_jobs = AutoField() # id - # DETAIL: Key (relname, relnamespace)=(tab_jobs_id_job_seq, 16386) ' - # 'already exists. + id = AutoField() resource_id = CharField(null=True) # benötigt? diff --git a/src/actinia_parallel_plugin/resources/config.py b/src/actinia_parallel_plugin/resources/config.py index 01b74d0..f6d9841 100644 --- a/src/actinia_parallel_plugin/resources/config.py +++ b/src/actinia_parallel_plugin/resources/config.py @@ -54,7 +54,7 @@ class JOBTABLE: pw = 'gis' schema = 'actinia' table = 'tab_jobs' - id_field = 'idpk_jobs' + id_field = 'id' batch_id_field = "batch_id" resource_id_field = "resource_id" diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 7b01070..83905fc 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -34,7 +34,7 @@ resource_id2 = "resource_id-291e8428-ec86-40a6-9ed2-ef1e14357aff" jobs = [ { - "idpk_jobs": 7, + "id": 7, "process": "ephemeral", "rule_configuration": { "list": [{"..."}], @@ -72,7 +72,7 @@ }, }, { - "idpk_jobs": 8, + "id": 8, "process": "ephemeral", "rule_configuration": { "list": [{"..."}], @@ -110,7 +110,7 @@ }, }, { - "idpk_jobs": 5, + "id": 5, "process": "ephemeral", "rule_configuration": { "list": [{"..."}], @@ -182,7 +182,7 @@ }, }, { - "idpk_jobs": 6, + "id": 6, "process": "ephemeral", "rule_configuration": { "list": [{"..."}], From c978d66ec33a88c8af867daeb70e3de05cea7792 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 11:19:54 +0200 Subject: [PATCH 38/47] remove some job descriptions --- .../batchjob_post_response_example.json | 111 ------------------ .../jobs_get_docs_response_example.json | 111 ------------------ src/actinia_parallel_plugin/apidocs/jobs.py | 4 - .../apidocs/regeldatei.py | 39 ------ src/actinia_parallel_plugin/core/batches.py | 7 -- src/actinia_parallel_plugin/core/jobtable.py | 2 - src/actinia_parallel_plugin/model/batch.py | 1 - .../model/batch_process_chain.py | 6 +- .../model/jobtabelle.py | 7 +- tests/unittests/test_batches.py | 100 +--------------- 10 files changed, 8 insertions(+), 380 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index ea581e5..07c7c51 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -33,117 +33,6 @@ } }, "actinia_gdi_batchid": 5, - "batch_description": { - "jobs": [ - { - "list": [ - { - "id": "g_region_nonparallel_block1", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_mapcalc_0_nonparallel_block1", - "inputs": [ - { - "param": "expression", - "value": "baum = elevation@PERMANENT * 2" - } - ], - "module": "r.mapcalc" - } - ], - "parallel": "false", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_1_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_info_1_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elevation@PERMANENT" - } - ], - "module": "r.info" - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "import_descr": { - "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", - "type": "raster" - }, - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region" - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_nonparallel_block3", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - } - ], - "parallel": "false", - "version": "1" - } - ] - }, "creation_uuids": [ "768247dd-8dce-4aa2-bf96-51fb698e4fea", "9163ca23-aad9-48c6-8eb4-80cc174d2090", diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index e944119..8ee3dce 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -111,117 +111,6 @@ }, "user_id": "actinia-gdi" }, - "batch_description": { - "jobs": [ - { - "list": [ - { - "id": "g_region_nonparallel_block1", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_mapcalc_0_nonparallel_block1", - "inputs": [ - { - "param": "expression", - "value": "baum = elevation@PERMANENT * 2" - } - ], - "module": "r.mapcalc" - } - ], - "parallel": "false", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_1_parallel_block2", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - }, - { - "id": "r_info_1_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elevation@PERMANENT" - } - ], - "module": "r.info" - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_2_parallel_block2", - "inputs": [ - { - "import_descr": { - "source": "https://apps.mundialis.de/actinia_test_datasets/elev_ned_30m.tif", - "type": "raster" - }, - "param": "raster", - "value": "elev_ned_30m" - } - ], - "module": "g.region" - }, - { - "flags": "g", - "id": "r_univar_2_parallel_block2", - "inputs": [ - { - "param": "map", - "value": "elev_ned_30m" - } - ], - "module": "r.univar", - "stdout": { - "delimiter": "=", - "format": "kv", - "id": "stats" - } - } - ], - "parallel": "true", - "version": "1" - }, - { - "list": [ - { - "flags": "p", - "id": "g_region_nonparallel_block3", - "inputs": [ - { - "param": "raster", - "value": "elevation@PERMANENT" - } - ], - "module": "g.region" - } - ], - "parallel": "false", - "version": "1" - } - ] - }, "batch_id": 3, "creation_uuid": "49cdd55a-332b-4f66-ad40-c0b3e576f824", "id": 11, diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index 5bde0bb..cc1ed8a 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -28,9 +28,6 @@ import json from flask_restful_swagger_2 import Schema -from actinia_parallel_plugin.apidocs.regeldatei import ( - RegeldateiModel, -) script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -53,7 +50,6 @@ class ProcessesJobResponseModel(Schema): 'description': 'The process of the job, e.g standortsicherung ' 'or potentialtrenches' }, - 'rule_configuration': RegeldateiModel, 'time_created': { 'type': 'string', 'description': 'Timestamp when job was created' diff --git a/src/actinia_parallel_plugin/apidocs/regeldatei.py b/src/actinia_parallel_plugin/apidocs/regeldatei.py index 8994224..d7e840e 100644 --- a/src/actinia_parallel_plugin/apidocs/regeldatei.py +++ b/src/actinia_parallel_plugin/apidocs/regeldatei.py @@ -262,42 +262,3 @@ class ProcessesProcModel(Schema): ' depends on. See also "status" as input parameter' } } - - -class RegeldateiModel(Schema): - """Request schema to create a job""" - # TODO check if this is correct - type = 'object' - properties = { - 'rule_area_id': { - 'type': 'integer', - 'description': 'Identifier of area where Regeldatei is valid' - }, - 'rule_area': { - 'type': 'string', - 'description': 'Name of area where Regeldatei is valid' - }, - 'feature_uuid': { - 'type': 'string', - 'description': 'Geonetwork UUID of feature type to run job with' - }, - 'feature_source': ProcessesProcInputModel, - 'processing_platform': { - 'type': 'string', - 'description': 'TODO' - }, - 'processing_platform_name': { - 'type': 'string', - 'description': 'TODO (and a unique ID.)' - }, - 'processing_host': { - 'type': 'string', - 'description': 'TODO (The actinia-core IP or URL)' - }, - 'procs': { - 'type': 'array', - 'description': 'List of processes to run', - 'items': ProcessesProcModel - } - } - required = ["feature_source", "procs"] diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 2c2b1c9..bcf0540 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -132,15 +132,9 @@ def createBatch(jsonDict, process, batchid): else: jobs_in_db = [] for job in jobs: - job["batch_description"] = jsonDict - # job["processing_host"] = jsonDict["processing_host"] - # job["processing_platform_name"] = jsonDict[ - # "processing_platform_name"] job["batch_id"] = batchid # assign the model process_chain = SingleJob(**job) - # might be needed (?) - # process_chain.feature_type = "null" job_in_db = insertJob(job, process, process_chain) jobs_in_db.append(job_in_db) return jobs_in_db @@ -246,7 +240,6 @@ def createBatchResponseDict(jobs_list): "actinia_core_response": responses, "creation_uuids": uuids, "id": job_ids, - "batch_description": jobs[0]["batch_description"], "process": process, "jobs_status": jobs_status, "status": batch_status diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 09e4c3c..cf8c2cc 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -255,8 +255,6 @@ def insertNewJob( # then it's a batch job job_kwargs["batch_processing_block"] = rule_configuration["batch_processing_block"] job_kwargs["batch_id"] = rule_configuration["batch_id"] - job_kwargs["batch_description"] = rule_configuration[ - "batch_description"] job = Job(**job_kwargs) with jobdb: diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 21d0e6a..cd62a32 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -178,7 +178,6 @@ class BatchJobResponseModel(Schema): 'type': 'integer', 'description': 'The batch ID' }, - 'batch_description': BatchProcessChainModel, 'creation_uuid': { 'type': 'array', 'description': ('Unique ids for the individual jobs at creation ' diff --git a/src/actinia_parallel_plugin/model/batch_process_chain.py b/src/actinia_parallel_plugin/model/batch_process_chain.py index 7a4b056..339ea99 100644 --- a/src/actinia_parallel_plugin/model/batch_process_chain.py +++ b/src/actinia_parallel_plugin/model/batch_process_chain.py @@ -121,9 +121,7 @@ class SingleJob(models.Base): processing_platform = fields.StringField() # string processing_platform_name = fields.StringField() # string processing_host = fields.StringField() # string - # batch_processing_block, batch_description and - # batch id are not in the json but is filled later + # batch_processing_block and batch_id are not in the json but is + # filled later batch_processing_block = fields.IntField() batch_id = fields.IntField() - # batch_description holds the entire batch processing chain - batch_description = fields.EmbeddedField(BatchProcessChain) diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 9a2546e..4207f1a 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -75,15 +75,12 @@ class Job(BaseModel): # benötigt? process = CharField(null=True) - # message = CharField(null=True) - # weg - rule_configuration = BinaryJSONField(null=True) # WIRD VERWENDET!!! vielleicht eher batch_description löschen? - batch_description = BinaryJSONField(null=True) # weg als eintrag + rule_configuration = BinaryJSONField(null=True) # add a potential parent_job batch_id = IntegerField(null=True) - batch_processing_block = IntegerField(null=True) # batch_batch_processing_block + batch_processing_block = IntegerField(null=True) class Meta: table_name = JOBTABLE.table diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 83905fc..b721285 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -36,21 +36,6 @@ { "id": 7, "process": "ephemeral", - "rule_configuration": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "parallel": "true", - "batch_processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 930771), "time_started": None, "time_estimated": None, @@ -61,34 +46,11 @@ "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", "message": None, "batch_id": 2, - "batch_processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, + "batch_processing_block": 2 }, { "id": 8, "process": "ephemeral", - "rule_configuration": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "parallel": "false", - "batch_processing_block": 3, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 940472), "time_started": None, "time_estimated": None, @@ -99,34 +61,11 @@ "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", "message": None, "batch_id": 2, - "batch_processing_block": 3, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, + "batch_processing_block": 3 }, { "id": 5, "process": "ephemeral", - "rule_configuration": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "parallel": "false", - "batch_processing_block": 1, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 906827), "time_started": None, "time_estimated": None, @@ -171,34 +110,11 @@ "creation_uuid": "767596e2-a9e4-4e96-b05b-4d77ae304a54", "message": None, "batch_id": 2, - "batch_processing_block": 1, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, + "batch_processing_block": 1 }, { "id": 6, "process": "ephemeral", - "rule_configuration": { - "list": [{"..."}], - "version": "1", - "batch_id": 2, - "parallel": "true", - "batch_processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, - }, "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 920875), "time_started": None, "time_estimated": None, @@ -243,15 +159,7 @@ "creation_uuid": "d08c1bbb-72f4-482f-bc78-672756937efa", "message": None, "batch_id": 2, - "batch_processing_block": 2, - "batch_description": { - "jobs": [ - {"list": [{"..."}], "version": "1", "parallel": "false"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "true"}, - {"list": [{"..."}], "version": "1", "parallel": "false"}, - ] - }, + "batch_processing_block": 2 }, ] block = [1, 2, 3] From 8ad076f680d520f312e0c7736f9b21a24438f488 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 11:28:50 +0200 Subject: [PATCH 39/47] rename actinia_core_response to resource_response --- .../apidocs/examples/batchjob_post_response_example.json | 2 +- .../apidocs/examples/jobs_get_docs_response_example.json | 2 +- src/actinia_parallel_plugin/apidocs/jobs.py | 2 +- src/actinia_parallel_plugin/core/batches.py | 4 ++-- src/actinia_parallel_plugin/core/jobtable.py | 8 ++++---- src/actinia_parallel_plugin/model/batch.py | 2 +- src/actinia_parallel_plugin/model/jobtabelle.py | 2 +- .../test_parallel_ephemeral_processing.py | 8 ++++---- tests/unittests/test_batches.py | 8 ++++---- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index 07c7c51..daafb20 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -5,7 +5,7 @@ "None", "None" ], - "actinia_core_response": { + "resource_response": { "resource_id-f5c2ec5d-30f0-4f51-be9a-5b423fa1eb18": { "accept_datetime": "2022-05-24 18:22:42.343953", "accept_timestamp": 1653416562.3439503, diff --git a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json index 8ee3dce..cf7dda6 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/jobs_get_docs_response_example.json @@ -1,6 +1,6 @@ { "resource_id": "resource_id-71264d56-4183-4d68-8544-5425254f5def", - "actinia_core_response": { + "resource_response": { "accept_datetime": "2022-05-24 18:22:12.481060", "accept_timestamp": 1653416532.4810588, "api_info": { diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index cc1ed8a..dc6c112 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -77,7 +77,7 @@ class ProcessesJobResponseModel(Schema): "TERMINATED" ] }, - 'actinia_core_response': { + 'resource_response': { 'type': 'object', 'description': 'The Response of actinia-core at creation time' }, diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index bcf0540..9e9abfd 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -175,7 +175,7 @@ def createBatchResponseDict(jobs_list): # this way we also have "None" if no resource_id is given yet: resource_ids.append(str(resource_id)) if resource_id is not None: - responses[resource_id] = job["actinia_core_response"] + responses[resource_id] = job["resource_response"] job_status = { "id": job["id"], "resource_id": str(resource_id), @@ -237,7 +237,7 @@ def createBatchResponseDict(jobs_list): "actinia_gdi_batchid": batch_id, "resource_id": resource_ids, "summary": summary_dict, - "actinia_core_response": responses, + "resource_response": responses, "creation_uuids": uuids, "id": job_ids, "process": process, diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index cf8c2cc..031a359 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -315,7 +315,7 @@ def updateJobByID( + str(record['id']) + ".") updatekwargs = { 'status': status, - 'actinia_core_response': resp, + 'resource_response': resp, 'resource_id': resourceId } @@ -327,13 +327,13 @@ def updateJobByID( updatekwargs = dict() if dbStatus == status: - updatekwargs['actinia_core_response'] = resp + updatekwargs['resource_response'] = resp else: log.debug("Update status to " + status + " for job with id " + str(record['id']) + ".") updatekwargs['status'] = status - updatekwargs['actinia_core_response'] = resp + updatekwargs['resource_response'] = resp updatekwargs['time_started'] = utcnow if resourceId is not None: updatekwargs['resource_id'] = resourceId @@ -349,7 +349,7 @@ def updateJobByID( + str(record['id']) + ".") updatekwargs = { 'status': status, - 'actinia_core_response': resp, + 'resource_response': resp, 'time_ended': utcnow } if resourceId is not None: diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index cd62a32..31b7d12 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -169,7 +169,7 @@ class BatchJobResponseModel(Schema): 'jobs'), 'items': {'type': 'string'} }, - 'actinia_core_response': { + 'resource_response': { 'type': 'array', 'description': 'The responses of actinia-core for individual jobs', 'items': {'type': 'object'} diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index 4207f1a..e104c3d 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -69,7 +69,7 @@ class Job(BaseModel): status = CharField(null=True) creation_uuid = CharField(null=True) - actinia_core_response = BinaryJSONField(null=True) # resource_response !!! + resource_response = BinaryJSONField(null=True) # resource_response !!! id = AutoField() resource_id = CharField(null=True) diff --git a/tests/integrationtests/test_parallel_ephemeral_processing.py b/tests/integrationtests/test_parallel_ephemeral_processing.py index fd18f8d..a03ae76 100644 --- a/tests/integrationtests/test_parallel_ephemeral_processing.py +++ b/tests/integrationtests/test_parallel_ephemeral_processing.py @@ -138,12 +138,12 @@ def test_post_parallel_ephemeral_processing(self): http_status=200, status="SUCCESS", ) - assert "actinia_core_response" in resp, \ - "No 'actinia_core_response' in response" - assert len(resp["actinia_core_response"]) == 4, \ + assert "resource_response" in resp, \ + "No 'resource_response' in response" + assert len(resp["resource_response"]) == 4, \ "There are not 4 actinia core responses" process_results = [ ac_resp["process_results"] for key, ac_resp in - resp["actinia_core_response"].items() if + resp["resource_response"].items() if ac_resp["process_results"] != {}] assert "stats" in process_results[0] diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index b721285..859ec4c 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -41,7 +41,7 @@ "time_estimated": None, "time_ended": None, "status": "PREPARING", - "actinia_core_response": None, + "resource_response": None, "resource_id": None, "creation_uuid": "81eae975-62c1-46f1-97d3-e027834a11b8", "message": None, @@ -56,7 +56,7 @@ "time_estimated": None, "time_ended": None, "status": "PREPARING", - "actinia_core_response": None, + "resource_response": None, "resource_id": None, "creation_uuid": "a4be1541-70cc-42ac-b134-c17f0ea8d311", "message": None, @@ -71,7 +71,7 @@ "time_estimated": None, "time_ended": datetime.datetime(2022, 6, 2, 7, 20, 15), "status": "SUCCESS", - "actinia_core_response": { + "resource_response": { "urls": { "status": f"{baseurl}/resources/actinia-gdi/{resource_id2}", "resources": [], @@ -120,7 +120,7 @@ "time_estimated": None, "time_ended": datetime.datetime(2022, 6, 2, 7, 20, 49), "status": "SUCCESS", - "actinia_core_response": { + "resource_response": { "urls": { "status": f"{baseurl}/resources/actinia-gdi/{resource_id1}", "resources": [], From 2b4efebf6b426a57f6f25d0653fabd832054a7a7 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 14:12:34 +0200 Subject: [PATCH 40/47] remove process --- .../api/parallel_ephemeral_processing.py | 4 +-- .../batchjob_post_response_example.json | 1 - src/actinia_parallel_plugin/apidocs/jobs.py | 7 +----- src/actinia_parallel_plugin/core/batches.py | 25 +++++++++---------- .../core/ephemeral_processing.py | 5 +--- src/actinia_parallel_plugin/core/jobs.py | 6 ++--- src/actinia_parallel_plugin/core/jobtable.py | 11 ++------ src/actinia_parallel_plugin/model/batch.py | 4 --- .../model/jobtabelle.py | 7 +----- tests/unittests/test_batches.py | 4 --- 10 files changed, 21 insertions(+), 53 deletions(-) diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index 62a43e6..3191075 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -83,7 +83,7 @@ def post(self, location_name): # assign new batchid self.batch_id = createBatchId() # create processing blocks and insert jobs into jobtable - jobs_in_db = createBatch(json_dict, "ephemeral", self.batch_id) + jobs_in_db = createBatch(json_dict, self.batch_id) if jobs_in_db is None: res = (jsonify(SimpleResponseModel( status=500, @@ -121,7 +121,7 @@ def post(self, location_name): "ephemeral" ) first_status = [entry["status"] for entry in first_jobs] - all_jobs = getJobsByBatchId(self.batch_id, "ephemeral") + all_jobs = getJobsByBatchId(self.batch_id) if None in first_jobs: res = (jsonify(SimpleResponseModel( status=500, diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index daafb20..f18b2da 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -67,7 +67,6 @@ "status": "PREPARING" } ], - "process": "ephemeral", "status": "PENDING", "summary": { "blocks": [ diff --git a/src/actinia_parallel_plugin/apidocs/jobs.py b/src/actinia_parallel_plugin/apidocs/jobs.py index dc6c112..2f2779c 100644 --- a/src/actinia_parallel_plugin/apidocs/jobs.py +++ b/src/actinia_parallel_plugin/apidocs/jobs.py @@ -45,11 +45,6 @@ class ProcessesJobResponseModel(Schema): 'type': 'integer', 'description': 'The job ID' }, - 'process': { - 'type': 'string', - 'description': 'The process of the job, e.g standortsicherung ' - 'or potentialtrenches' - }, 'time_created': { 'type': 'string', 'description': 'Timestamp when job was created' @@ -79,7 +74,7 @@ class ProcessesJobResponseModel(Schema): }, 'resource_response': { 'type': 'object', - 'description': 'The Response of actinia-core at creation time' + 'description': 'The Response at creation time' }, 'resource_id': { 'type': 'string', diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 9e9abfd..1f118cc 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -79,11 +79,11 @@ def assignProcessingBlocks(jsonDict): return result_jobs -# def cancelBatch(batchid, process): +# def cancelBatch(batchid): # """ Function to cancel all jobs that are RUNNING or PENDING -# by batchid and process +# by batchid # """ -# jobs = getJobsByBatchId(batchid, process) +# jobs = getJobsByBatchId(batchid) # cancel_jobs = [] # id_field = JOBTABLE.id_field # for job in jobs: @@ -95,7 +95,7 @@ def assignProcessingBlocks(jsonDict): # else: # # then there are jobs that have not been posted to actinia yet, # # where cancelJob returns None only -# return getJobsByBatchId(batchid, process) +# return getJobsByBatchId(batchid) def checkBatchProcessChain(jsonDict): @@ -123,7 +123,7 @@ def checkProcessingBlockFinished(jobs, block): return finished -def createBatch(jsonDict, process, batchid): +def createBatch(jsonDict, batchid): """ Function to insert all jobs from a batch into the joblist """ jobs = assignProcessingBlocks(jsonDict) @@ -135,7 +135,7 @@ def createBatch(jsonDict, process, batchid): job["batch_id"] = batchid # assign the model process_chain = SingleJob(**job) - job_in_db = insertJob(job, process, process_chain) + job_in_db = insertJob(job, process_chain) jobs_in_db.append(job_in_db) return jobs_in_db @@ -162,7 +162,6 @@ def createBatchResponseDict(jobs_list): # sort the jobs according to their id jobs = sorted(jobs_list, key=lambda d: d["id"]) - process = jobs[0]["process"] batch_id = jobs[0]["batch_id"] resource_ids = [] uuids = [] @@ -240,7 +239,7 @@ def createBatchResponseDict(jobs_list): "resource_response": responses, "creation_uuids": uuids, "id": job_ids, - "process": process, + # "process": process, "jobs_status": jobs_status, "status": batch_status } @@ -255,25 +254,25 @@ def getAllBatchIds(): return batch_ids -# def getAllBatches(process): +# def getAllBatches(): # """ Function to return all jobs that are part of a batch from the # database # """ # result_list = [] # batchids = getAllBatchIds() # for batchid in batchids: -# jobs = getJobsByBatchId(batchid, process) +# jobs = getJobsByBatchId(batchid) # jobs_response = createBatchResponseDict(jobs) # result_list.append(jobs_response) # result_dict = {"batch_jobs": result_list} # return result_dict -def getJobsByBatchId(batch_id, process=None): - """ Function to return all jobs (db entries) via a batch_id and process +def getJobsByBatchId(batch_id): + """ Function to return all jobs (db entries) via a batch_id """ filter_dict = {"batch_id": batch_id} - jobs = getAllJobs(filter_dict, process) + jobs = getAllJobs(filter_dict) return jobs diff --git a/src/actinia_parallel_plugin/core/ephemeral_processing.py b/src/actinia_parallel_plugin/core/ephemeral_processing.py index bf0672b..bef60cf 100644 --- a/src/actinia_parallel_plugin/core/ephemeral_processing.py +++ b/src/actinia_parallel_plugin/core/ephemeral_processing.py @@ -73,10 +73,7 @@ def _update_and_check_batch_jobs(self): updateJob(resource_id, response_model, self.jobid) if "finished" == response_model["status"]: - jobs_from_batch = getJobsByBatchId( - self.batch_id, - "ephemeral" - ) + jobs_from_batch = getJobsByBatchId(self.batch_id) all_blocks = [ job["batch_processing_block"] for job in jobs_from_batch] block = int(self.batch_processing_block) diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index 7b30709..421671e 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -Module to start the process Standortsicherung +Functions to communicate with the job db """ __license__ = "GPLv3" @@ -32,7 +32,7 @@ from actinia_parallel_plugin.resources.logging import log -def insertJob(jsonDict, process, process_chain): +def insertJob(jsonDict, process_chain): """ function to prepare and call InsertNewJob from regeldatei""" try: @@ -44,7 +44,6 @@ def insertJob(jsonDict, process, process_chain): job = insertNewJob( jsonDict, - process, ) return job @@ -79,7 +78,6 @@ def updateJob(resource_id, actinia_resp, jobid): """ status = actinia_resp["status"] - # record = getJobByResource("resource_id", resource_id) # follow-up actinia update, therefore without resourceId record = updateJobByID( diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 031a359..7bf3b42 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -92,7 +92,7 @@ def getAllIds(batch=False): return jobIds -def getAllJobs(filters, process=None): +def getAllJobs(filters): """ Method to read all jobs from jobtabelle with filter Args: filters (ImmutableMultiDict): the args from the HTTP call @@ -102,12 +102,7 @@ def getAllJobs(filters, process=None): """ log.debug('Received query for jobs') - if process == 'test': - query = Expression('a', '=', 'a') - elif process is None: - query = None - else: - query = Expression(getattr(Job, 'process'), '=', process) + query = None if filters: log.debug("Found filters: " + str(filters)) @@ -229,7 +224,6 @@ def getJobByResource(key, val): def insertNewJob( rule_configuration, - process, ): """Insert new job into jobtabelle. @@ -246,7 +240,6 @@ def insertNewJob( job_kwargs = { 'rule_configuration': rule_configuration, - 'process': process, 'status': 'PREPARING', 'time_created': utcnow, 'creation_uuid': creation_uuid diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 31b7d12..aba8d52 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -190,10 +190,6 @@ class BatchJobResponseModel(Schema): 'description': 'The individual job IDs', 'items': {'type': 'integer'} }, - 'process': { - 'type': 'string', - 'description': 'The process of the job, e.g. netdefinition' - }, 'status': { 'type': 'string', 'description': 'The overall status of the batchjob', diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtabelle.py index e104c3d..26a3f99 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtabelle.py @@ -68,14 +68,9 @@ class Job(BaseModel): time_ended = DateTimeField(null=True) status = CharField(null=True) creation_uuid = CharField(null=True) - - resource_response = BinaryJSONField(null=True) # resource_response !!! + resource_response = BinaryJSONField(null=True) id = AutoField() resource_id = CharField(null=True) - - # benötigt? - process = CharField(null=True) - rule_configuration = BinaryJSONField(null=True) # add a potential parent_job diff --git a/tests/unittests/test_batches.py b/tests/unittests/test_batches.py index 859ec4c..9f29df9 100644 --- a/tests/unittests/test_batches.py +++ b/tests/unittests/test_batches.py @@ -35,7 +35,6 @@ jobs = [ { "id": 7, - "process": "ephemeral", "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 930771), "time_started": None, "time_estimated": None, @@ -50,7 +49,6 @@ }, { "id": 8, - "process": "ephemeral", "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 940472), "time_started": None, "time_estimated": None, @@ -65,7 +63,6 @@ }, { "id": 5, - "process": "ephemeral", "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 906827), "time_started": None, "time_estimated": None, @@ -114,7 +111,6 @@ }, { "id": 6, - "process": "ephemeral", "time_created": datetime.datetime(2022, 6, 2, 7, 20, 14, 920875), "time_started": None, "time_estimated": None, From 8c7bf165e6a84ff6fa72839835d2016435928ff7 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 14:20:33 +0200 Subject: [PATCH 41/47] renaming jobtabelle.py --- src/actinia_parallel_plugin/core/jobtable.py | 14 +++++++------- src/actinia_parallel_plugin/model/batch.py | 6 ------ .../model/{jobtabelle.py => jobtable.py} | 2 +- src/actinia_parallel_plugin/resources/config.py | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) rename src/actinia_parallel_plugin/model/{jobtabelle.py => jobtable.py} (99%) diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index 7bf3b42..bdb8d05 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -34,7 +34,7 @@ from yoyo import read_migrations from yoyo import get_backend -from actinia_parallel_plugin.model.jobtabelle import Job, jobdb +from actinia_parallel_plugin.model.jobtable import Job, jobdb from actinia_parallel_plugin.resources.config import JOBTABLE from actinia_parallel_plugin.resources.logging import log @@ -64,7 +64,7 @@ def applyMigrations(): def getAllIds(batch=False): - """ Method to read all jobs from jobtabelle + """ Method to read all jobs from jobtable Args: batch (bool): indicate whether only batch jobs should be read @@ -93,7 +93,7 @@ def getAllIds(batch=False): def getAllJobs(filters): - """ Method to read all jobs from jobtabelle with filter + """ Method to read all jobs from jobtable with filter Args: filters (ImmutableMultiDict): the args from the HTTP call @@ -154,7 +154,7 @@ def getAllJobs(filters): def getJobById(jobid): - """ Method to read job from jobtabelle by id + """ Method to read job from jobtable by id Args: jobid (int): id of job @@ -199,7 +199,7 @@ def getJobById(jobid): def getJobByResource(key, val): - """ Method to read job from jobtabelle by resource + """ Method to read job from jobtable by resource Args: key (string): key of attribute @@ -225,7 +225,7 @@ def getJobByResource(key, val): def insertNewJob( rule_configuration, ): - """Insert new job into jobtabelle. + """Insert new job into jobtable. Args: rule_configuration (dict): original regeldatei @@ -269,7 +269,7 @@ def insertNewJob( def updateJobByID( jobid, status, resp, resourceId=None): - """ Method to update job in jobtabelle when processing status changed + """ Method to update job in jobtable when processing status changed Args: jobid (int): the id of the job diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index aba8d52..1ac5fe8 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -141,12 +141,6 @@ class BatchProcessChainModel(Schema): """ type = 'object' properties = { - # 'processing_host': { - # 'type': 'string', - # 'description': 'The actinia-core IP or URL in case the platform ' - # 'is not OpenShift and no new VM should be created' - # ' by actinia-gdi' - # }, 'jobs': { 'type': 'array', 'items': ProcessChainModel, diff --git a/src/actinia_parallel_plugin/model/jobtabelle.py b/src/actinia_parallel_plugin/model/jobtable.py similarity index 99% rename from src/actinia_parallel_plugin/model/jobtabelle.py rename to src/actinia_parallel_plugin/model/jobtable.py index 26a3f99..fc356f6 100644 --- a/src/actinia_parallel_plugin/model/jobtabelle.py +++ b/src/actinia_parallel_plugin/model/jobtable.py @@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -Response models +Job model """ __license__ = "GPLv3" diff --git a/src/actinia_parallel_plugin/resources/config.py b/src/actinia_parallel_plugin/resources/config.py index f6d9841..dea1dfa 100644 --- a/src/actinia_parallel_plugin/resources/config.py +++ b/src/actinia_parallel_plugin/resources/config.py @@ -45,7 +45,7 @@ class JOBTABLE: - """Default config for database connection for jobtabelle + """Default config for database connection for jobtable """ host = 'localhost' port = '5432' From 2c739188ea37902f1092c894bcd272c438495c64 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 14:55:07 +0200 Subject: [PATCH 42/47] flake8 --- .../apidocs/regeldatei.py | 264 ------------------ src/actinia_parallel_plugin/core/batches.py | 5 +- src/actinia_parallel_plugin/core/jobs.py | 16 +- src/actinia_parallel_plugin/core/jobtable.py | 9 +- .../core/parallel_resource_base.py | 1 - .../model/batch_process_chain.py | 22 -- 6 files changed, 8 insertions(+), 309 deletions(-) delete mode 100644 src/actinia_parallel_plugin/apidocs/regeldatei.py diff --git a/src/actinia_parallel_plugin/apidocs/regeldatei.py b/src/actinia_parallel_plugin/apidocs/regeldatei.py deleted file mode 100644 index d7e840e..0000000 --- a/src/actinia_parallel_plugin/apidocs/regeldatei.py +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2018-2022 mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Model classes for regeldatei -""" - -__license__ = "GPLv3" -__author__ = "Carmen Tawalika, Anika Weinmann" -__copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - -# import os -# import json - -from flask_restful_swagger_2 import Schema - -# script_dir = os.path.dirname(os.path.abspath(__file__)) -# null = "null" -# -# rel_path = ("../apidocs/examples/" + -# "standortsicherung_jobs_post_request_example.json") -# abs_file_path = os.path.join(script_dir, rel_path) -# with open(abs_file_path) as jsonfile: -# sos_jobs_post_docs_request_example = json.load(jsonfile) -# -# rel_path = ("../apidocs/examples/" + -# "pt_jobs_post_request_example.json") -# abs_file_path = os.path.join(script_dir, rel_path) -# with open(abs_file_path) as jsonfile: -# pt_jobs_post_docs_request_example = json.load(jsonfile) - - -class ProcessesProcInputBaseModel(Schema): - type = 'object' - properties = { - 'name': { - 'type': 'string', - 'description': 'Name of input data' - }, - 'type': { - 'type': 'string', - 'enum': ["GNOS", "DATABASE", "PARAMETER", "STATUS"], - 'description': 'Type of input. Can be "GNOS", "DATABASE", ' - '"PARAMETER" or "STATUS"' - } - } - - -class ProcessesFilterModel(Schema): - type = 'object' - properties = { - 'name': { - 'type': 'string', - 'description': 'Not yet implemented or specified' - }, - # 'metadata_field': { - # 'type': 'string', - # 'description': 'Not yet implemented or specified' - # }, - # 'metadata_value': { - # 'type': 'string', - # 'description': 'Not yet implemented or specified' - # }, - 'operator': { - 'type': 'string', - 'description': 'Not yet implemented or specified' - } - } - - -class ProcessesProcInputDatabaseModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - }, - { - 'type': 'object', - 'properties': { - 'table': { - 'type': 'string', - 'description': 'Database connection string with table' - }, - 'attributes': { - 'type': 'array', - 'description': 'Database attribute of data source to use', - 'items': { - 'type': 'string' - } - }, - 'filter': { - 'type': 'array', - 'description': 'Not yet implemented or specified', - 'items': ProcessesFilterModel - } - } - } - ] - - -class ProcessesProcInputParameterModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - }, - { - 'type': 'object', - 'properties': { - 'value': { - 'type': 'array', - 'description': 'Array of input parameter. Can be int, ' - 'float or string', - 'items': { - # TODO: find out how to allow multiple types - # 'type': 'int, float, string' - } - }, - 'uom': { - 'type': 'string', - 'description': 'Unit of measurement for values' - } - } - } - ] - - -class ProcessesProcInputFileModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - }, - { - 'type': 'object', - 'properties': { - 'value': { - 'type': 'array', - 'description': 'Array of input parameter. Can be int, ' - 'float or string', - 'items': { - # TODO: find out how to allow multiple types - # 'type': 'int, float, string' - } - } - } - } - ] - - -class ProcessesProcInputStatusModel(Schema): - """Request schema for creating a job""" - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputBaseModel, - { - '$ref': '#/definitions/ProcessesProcInputBaseModel' - }, - { - 'type': 'object', - 'properties': { - 'status': { - 'type': 'string', - 'description': 'Status of another process as input. See ' - 'also "dependsOn"' - } - } - } - ] - - -class ProcessesProcInputModel(Schema): - """Definitions for process input data""" - type = 'object' - # TODO: use oneOf (was not parsed in petstore) - allOf = [ - # keep this line, otherwise BaseModel does not exist in document - ProcessesProcInputDatabaseModel, - ProcessesProcInputParameterModel, - ProcessesProcInputFileModel, - ProcessesProcInputStatusModel, - { - '$ref': '#/definitions/ProcessesProcInputDatabaseModel' - }, - { - '$ref': '#/definitions/ProcessesProcInputParameterModel' - }, - { - '$ref': '#/definitions/ProcessesProcInputFileModel' - }, - { - '$ref': '#/definitions/ProcessesProcInputStatusModel' - } - ] - - -class ProcessesProcOutputModel(Schema): - """Definitions for process output data""" - type = 'object' - properties = { - 'name': { - 'type': 'string', - 'description': 'Name of attribute column in output datasource' - }, - 'type': { - 'type': 'string', - 'description': 'Column type of attribut column' - }, - 'default': { - 'type': 'string', - 'description': 'Default value of attribut column' - }, - 'value': { - 'type': 'string', - 'description': 'Filename if output is file' - } - } - - -class ProcessesProcModel(Schema): - """List of processes to run""" - type = 'object' - properties = { - 'name': { - 'type': 'integer', - 'description': 'Name of process' - }, - 'input': { - 'type': 'array', - 'description': 'Definitions for process input data', - 'items': ProcessesProcInputModel - }, - 'output': { - 'type': 'array', - 'description': 'Definitions for process output data', - 'items': ProcessesProcOutputModel - }, - 'dependsOn': { - 'type': 'string', - 'description': 'List of names of processes on which this process' - ' depends on. See also "status" as input parameter' - } - } diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 1f118cc..50283b3 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -32,7 +32,6 @@ AsyncParallelJobResource from actinia_parallel_plugin.model.batch_process_chain import ( BatchProcessChain, - SingleJob, ) from actinia_parallel_plugin.resources.logging import log @@ -42,6 +41,7 @@ def assignProcessingBlocks(jsonDict): to the parallel parameter into processing blocks """ bpc_dict = checkBatchProcessChain(jsonDict) + if bpc_dict is None: return None else: @@ -134,8 +134,7 @@ def createBatch(jsonDict, batchid): for job in jobs: job["batch_id"] = batchid # assign the model - process_chain = SingleJob(**job) - job_in_db = insertJob(job, process_chain) + job_in_db = insertJob(job) jobs_in_db.append(job_in_db) return jobs_in_db diff --git a/src/actinia_parallel_plugin/core/jobs.py b/src/actinia_parallel_plugin/core/jobs.py index 421671e..924550f 100644 --- a/src/actinia_parallel_plugin/core/jobs.py +++ b/src/actinia_parallel_plugin/core/jobs.py @@ -29,22 +29,12 @@ insertNewJob, updateJobByID, ) -from actinia_parallel_plugin.resources.logging import log -def insertJob(jsonDict, process_chain): - """ function to prepare and call InsertNewJob from regeldatei""" +def insertJob(jsonDict): + """ function to prepare and call InsertNewJob""" - try: - process_chain_struct = process_chain.to_struct() - except Exception as e: - log.error('Regeldatei is invalid!') - log.error(e) - return None - - job = insertNewJob( - jsonDict, - ) + job = insertNewJob(jsonDict) return job diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index bdb8d05..b6122d8 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -85,8 +85,6 @@ def getAllIds(batch=False): for i in queryResult: jobIds.append(i[field]) - # log.debug("Information read from jobtable.") - jobdb.close() return jobIds @@ -246,7 +244,8 @@ def insertNewJob( } if "batch_id" in rule_configuration.keys(): # then it's a batch job - job_kwargs["batch_processing_block"] = rule_configuration["batch_processing_block"] + job_kwargs["batch_processing_block"] = rule_configuration[ + "batch_processing_block"] job_kwargs["batch_id"] = rule_configuration["batch_id"] job = Job(**job_kwargs) @@ -267,8 +266,7 @@ def insertNewJob( return record -def updateJobByID( - jobid, status, resp, resourceId=None): +def updateJobByID(jobid, status, resp, resourceId=None): """ Method to update job in jobtable when processing status changed Args: @@ -281,7 +279,6 @@ def updateJobByID( updatedRecord (TODO): the updated record """ - # TODO do not rename the status (kleineschreiben!!!) if status == 'accepted': status = 'PENDING' elif status == 'running': diff --git a/src/actinia_parallel_plugin/core/parallel_resource_base.py b/src/actinia_parallel_plugin/core/parallel_resource_base.py index 8e7be68..a5961b4 100644 --- a/src/actinia_parallel_plugin/core/parallel_resource_base.py +++ b/src/actinia_parallel_plugin/core/parallel_resource_base.py @@ -163,7 +163,6 @@ def preprocess(self, has_json=True, has_xml=False, # Create the resource URL base and use a placeholder for the file name # The placeholder __None__ must be replaced by the resource URL # generator - # TODO check if this works self.resource_url_base = f"{self.status_url}/__None__" if (global_config.FORCE_HTTPS_URLS is True diff --git a/src/actinia_parallel_plugin/model/batch_process_chain.py b/src/actinia_parallel_plugin/model/batch_process_chain.py index 339ea99..f81d46c 100644 --- a/src/actinia_parallel_plugin/model/batch_process_chain.py +++ b/src/actinia_parallel_plugin/model/batch_process_chain.py @@ -103,25 +103,3 @@ class BatchProcessChain(models.Base): processing_platform_name = fields.StringField() # string processing_host = fields.StringField() # string jobs = fields.ListField([Job], required=True) # array of objects - - -class SingleJob(models.Base): - """Model for SingleJob - Including all information for all modules and general information on - processing platform and host - This is used by the parallel processing netdefinition/batchjobs endpoint - and is only created internally for individual jobs of a BatchProcessChain - in order to have the processing_platform etc. attributes attached to each - job as well. - """ - version = fields.StringField() # string - list = fields.ListField([Module], required=True) # array of objects - # the following are given by the user to the BatchProcessChain and are - # then internally applied to each SingleJob - processing_platform = fields.StringField() # string - processing_platform_name = fields.StringField() # string - processing_host = fields.StringField() # string - # batch_processing_block and batch_id are not in the json but is - # filled later - batch_processing_block = fields.IntField() - batch_id = fields.IntField() From de43c6979ca1fd33d197c170896e451def4afa45 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 30 Jun 2022 15:06:29 +0200 Subject: [PATCH 43/47] review --- .../examples/batchjob_post_response_example.json | 2 +- src/actinia_parallel_plugin/core/batches.py | 2 +- src/actinia_parallel_plugin/model/batch.py | 2 +- .../model/batch_process_chain.py | 11 +++++------ tests/test_resource_base.py | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json index f18b2da..999f75b 100644 --- a/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json +++ b/src/actinia_parallel_plugin/apidocs/examples/batchjob_post_response_example.json @@ -32,7 +32,7 @@ "user_id": "actinia-gdi" } }, - "actinia_gdi_batchid": 5, + "batch_id": 5, "creation_uuids": [ "768247dd-8dce-4aa2-bf96-51fb698e4fea", "9163ca23-aad9-48c6-8eb4-80cc174d2090", diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 50283b3..47e8811 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -232,7 +232,7 @@ def createBatchResponseDict(jobs_list): # create overall response dict responseDict = { - "actinia_gdi_batchid": batch_id, + "batch_id": batch_id, "resource_id": resource_ids, "summary": summary_dict, "resource_response": responses, diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index 1ac5fe8..bc2b9bd 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -168,7 +168,7 @@ class BatchJobResponseModel(Schema): 'description': 'The responses of actinia-core for individual jobs', 'items': {'type': 'object'} }, - 'actinia_gdi_batchid': { + 'batch_id': { 'type': 'integer', 'description': 'The batch ID' }, diff --git a/src/actinia_parallel_plugin/model/batch_process_chain.py b/src/actinia_parallel_plugin/model/batch_process_chain.py index f81d46c..7932f52 100644 --- a/src/actinia_parallel_plugin/model/batch_process_chain.py +++ b/src/actinia_parallel_plugin/model/batch_process_chain.py @@ -94,12 +94,11 @@ class Job(models.Base): class BatchProcessChain(models.Base): """Model for BatchProcessChain - Including all information for all jobs and general information on - processing platform and host - This is used by the sequential processing netdefinition/jobs endpoint + Including all information for all jobs + This is used by the parallel processing endpoints """ - processing_platform = fields.StringField() # string - processing_platform_name = fields.StringField() # string - processing_host = fields.StringField() # string + # processing_platform = fields.StringField() # string + # processing_platform_name = fields.StringField() # string + # processing_host = fields.StringField() # string jobs = fields.ListField([Job], required=True) # array of objects diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index fc8e54e..4813202 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -140,7 +140,7 @@ def create_user(cls, name="guest", role="guest", def waitAsyncBatchJob(self, rv, headers, http_status=200, status="SUCCESS", message_check=None): resp_data = json_loads(rv.data) - batchid = resp_data["actinia_gdi_batchid"] + batchid = resp_data["batch_id"] while True: rv = self.server.get( From c5e5010e0f9001862e012a9613202a99f935cdf7 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 1 Jul 2022 09:52:44 +0200 Subject: [PATCH 44/47] config test --- .../actinia-parallel-plugin-test.cfg | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg index 7dc4aa9..98cb525 100644 --- a/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg +++ b/docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg @@ -31,4 +31,4 @@ database = gis user = actinia schema = actinia table = tab_jobs -id_field = idpk_jobs +id_field = id diff --git a/setup.cfg b/setup.cfg index 36c8197..1fe522b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,7 +76,7 @@ extras = True # in order to write a coverage file that can be read by Jenkins. addopts = --cov actinia_parallel_plugin --cov-report term-missing - --verbose --tb=line -x + --verbose --tb=line -x -s norecursedirs = dist build From 57dbe35fceb3f0de61e3741effee9d41ea272a8f Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Mon, 4 Jul 2022 09:59:17 +0200 Subject: [PATCH 45/47] changing endpoints --- README.md | 4 ++-- src/actinia_parallel_plugin/api/batch.py | 16 ++++++++++----- src/actinia_parallel_plugin/api/job.py | 19 +++++++++++++----- .../api/parallel_ephemeral_processing.py | 20 ++++++++++--------- src/actinia_parallel_plugin/core/batches.py | 10 +++++++--- src/actinia_parallel_plugin/core/jobtable.py | 2 ++ src/actinia_parallel_plugin/endpoints.py | 13 +++++++----- src/actinia_parallel_plugin/model/batch.py | 4 +++- src/actinia_parallel_plugin/model/jobtable.py | 1 + .../test_parallel_ephemeral_processing.py | 17 ++++++++++++++++ tests/test_resource_base.py | 9 ++++----- 11 files changed, 80 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index a29f9ab..447a9d6 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,9 @@ act -j unittests -P ubuntu-latest=lucasalt/act_base:latest ### Requesting batch job and job endpoints ``` # request batch job -curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel/batchjobs/1 | jq +curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/resources/actinia-gdi/batches/1 | jq # request job -curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel/jobs/1 | jq +curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/resources/actinia-gdi/batches/1/jobs/1 | jq ``` ### Start parallel batch job diff --git a/src/actinia_parallel_plugin/api/batch.py b/src/actinia_parallel_plugin/api/batch.py index 106f513..602f62e 100644 --- a/src/actinia_parallel_plugin/api/batch.py +++ b/src/actinia_parallel_plugin/api/batch.py @@ -24,12 +24,12 @@ __copyright__ = "Copyright 2021-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" -from flask_restful import Resource from flask_restful_swagger_2 import swagger from flask import make_response, jsonify from actinia_core.models.response_models import \ SimpleResponseModel +from actinia_core.rest.resource_management import ResourceManagerBase from actinia_parallel_plugin.resources.logging import log from actinia_parallel_plugin.core.batches import ( @@ -39,16 +39,22 @@ from actinia_parallel_plugin.apidocs import batch -class BatchJobsId(Resource): +class BatchJobsId(ResourceManagerBase): """ Definition for endpoint - @app.route('processing_parallel/batchjobs/') + @app.route('/resources//batches/') Contains HTTP GET endpoint Contains HTTP POST endpoint Contains swagger documentation """ @swagger.doc(batch.batchjobId_get_docs) - def get(self, batchid): + def get(self, user_id, batchid): + """Get the status of a batch.""" + + ret = self.check_permissions(user_id=user_id) + if ret: + return ret + if batchid is None: return make_response("No batchid was given", 400) @@ -69,7 +75,7 @@ def get(self, batchid): return make_response(jsonify(resp_dict), 200) # no docs because 405 - def post(self, batchid): + def post(self, user_id, batchid): res = jsonify(SimpleResponseModel( status=405, message="Method Not Allowed" diff --git a/src/actinia_parallel_plugin/api/job.py b/src/actinia_parallel_plugin/api/job.py index c3915a7..c6a0013 100644 --- a/src/actinia_parallel_plugin/api/job.py +++ b/src/actinia_parallel_plugin/api/job.py @@ -24,10 +24,10 @@ __copyright__ = "Copyright 2018-2022 mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH % Co. KG" -from flask_restful import Resource from flask_restful_swagger_2 import swagger from flask import make_response, jsonify +from actinia_core.rest.resource_management import ResourceManagerBase from actinia_core.models.response_models import \ SimpleResponseModel @@ -38,7 +38,7 @@ from actinia_parallel_plugin.apidocs import jobs -class JobId(Resource): +class JobId(ResourceManagerBase): """ Definition for endpoint standortsicherung @app.route('/processing_parallel/jobs/') @@ -47,13 +47,22 @@ class JobId(Resource): """ @swagger.doc(jobs.jobId_get_docs) - def get(self, jobid): + def get(self, user_id, batchid, jobid): """ Wrapper method to receive HTTP call and pass it to function This method is called by HTTP GET - @app.route('/processing_parallel/jobs/') + @app.route( + '/resources//batches//jobs/') This method is calling core method readJob """ + + ret = self.check_permissions(user_id=user_id) + if ret: + return ret + + if batchid is None: + return make_response("No batchid was given", 400) + if jobid is None: return make_response("No jobid was given", 400) @@ -70,7 +79,7 @@ def get(self, jobid): ))) return make_response(res, err["status"]) - def post(self, jobid): + def post(self, user_id, batchid, jobid): res = jsonify(SimpleResponseModel( status=405, message="Method Not Allowed" diff --git a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py index 3191075..6067936 100644 --- a/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py +++ b/src/actinia_parallel_plugin/api/parallel_ephemeral_processing.py @@ -82,15 +82,6 @@ def post(self, location_name): # assign new batchid self.batch_id = createBatchId() - # create processing blocks and insert jobs into jobtable - jobs_in_db = createBatch(json_dict, self.batch_id) - if jobs_in_db is None: - res = (jsonify(SimpleResponseModel( - status=500, - message=('Error: Batch Processing Chain JSON has no ' - 'jobs.') - ))) - return make_response(res, 500) # Generate the base of the status URL host_url = request.host_url @@ -104,6 +95,17 @@ def post(self, location_name): self.base_status_url = f"{host_url}{URL_PREFIX}/resources/" \ f"{g.user.user_id}/" + # create processing blocks and insert jobs into jobtable + status_url = f"{self.base_status_url}batches/{self.batch_id}" + jobs_in_db = createBatch(json_dict, self.batch_id, status_url) + if jobs_in_db is None: + res = (jsonify(SimpleResponseModel( + status=500, + message=('Error: Batch Processing Chain JSON has no ' + 'jobs.') + ))) + return make_response(res, 500) + # start first processing block first_jobs = startProcessingBlock( jobs_in_db, diff --git a/src/actinia_parallel_plugin/core/batches.py b/src/actinia_parallel_plugin/core/batches.py index 47e8811..f90f03c 100644 --- a/src/actinia_parallel_plugin/core/batches.py +++ b/src/actinia_parallel_plugin/core/batches.py @@ -123,7 +123,7 @@ def checkProcessingBlockFinished(jobs, block): return finished -def createBatch(jsonDict, batchid): +def createBatch(jsonDict, batchid, statusurl): """ Function to insert all jobs from a batch into the joblist """ jobs = assignProcessingBlocks(jsonDict) @@ -133,6 +133,7 @@ def createBatch(jsonDict, batchid): jobs_in_db = [] for job in jobs: job["batch_id"] = batchid + job["urls"] = {"status": statusurl, "resources": []} # assign the model job_in_db = insertJob(job) jobs_in_db.append(job_in_db) @@ -230,6 +231,9 @@ def createBatchResponseDict(jobs_list): "blocks": batch_processing_blocks } + # create urls + urls = jobs[0]["urls"] + # create overall response dict responseDict = { "batch_id": batch_id, @@ -238,9 +242,9 @@ def createBatchResponseDict(jobs_list): "resource_response": responses, "creation_uuids": uuids, "id": job_ids, - # "process": process, "jobs_status": jobs_status, - "status": batch_status + "status": batch_status, + "urls": urls, } return responseDict diff --git a/src/actinia_parallel_plugin/core/jobtable.py b/src/actinia_parallel_plugin/core/jobtable.py index b6122d8..941918a 100644 --- a/src/actinia_parallel_plugin/core/jobtable.py +++ b/src/actinia_parallel_plugin/core/jobtable.py @@ -247,6 +247,8 @@ def insertNewJob( job_kwargs["batch_processing_block"] = rule_configuration[ "batch_processing_block"] job_kwargs["batch_id"] = rule_configuration["batch_id"] + if "urls" in rule_configuration.keys(): + job_kwargs["urls"] = rule_configuration["urls"] job = Job(**job_kwargs) with jobdb: diff --git a/src/actinia_parallel_plugin/endpoints.py b/src/actinia_parallel_plugin/endpoints.py index 803a89a..5eb45ec 100644 --- a/src/actinia_parallel_plugin/endpoints.py +++ b/src/actinia_parallel_plugin/endpoints.py @@ -50,18 +50,21 @@ def create_endpoints(flask_api): # "/locations//mapsets/" # "/processing_parallel") + # GET batch jobs TODO + # "/resources//batches" + # GET batch jobs by ID apidoc.add_resource( BatchJobsId, - "/processing_parallel/batchjobs/") + "/resources//batches/") + + # GET all jobs of one batch TODO + # "/resources//batches//jobs" # GET job by ID apidoc.add_resource( JobId, - "/processing_parallel/jobs/") - - # "/processing_parallel/batchjobs" - # "/processing_parallel/jobs" + "/resources//batches//jobs/") # initilalize jobtable initJobDB() diff --git a/src/actinia_parallel_plugin/model/batch.py b/src/actinia_parallel_plugin/model/batch.py index bc2b9bd..a649bd3 100644 --- a/src/actinia_parallel_plugin/model/batch.py +++ b/src/actinia_parallel_plugin/model/batch.py @@ -29,6 +29,7 @@ from flask_restful_swagger_2 import Schema from actinia_core.models.process_chain import ProcessChainModel +from actinia_core.models.response_models import UrlModel script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -223,6 +224,7 @@ class BatchJobResponseModel(Schema): } } }, - 'summary': BatchJobsSummaryModel + 'summary': BatchJobsSummaryModel, + 'urls': UrlModel } example = batchjob_post_docs_response_example diff --git a/src/actinia_parallel_plugin/model/jobtable.py b/src/actinia_parallel_plugin/model/jobtable.py index fc356f6..b167026 100644 --- a/src/actinia_parallel_plugin/model/jobtable.py +++ b/src/actinia_parallel_plugin/model/jobtable.py @@ -72,6 +72,7 @@ class Job(BaseModel): id = AutoField() resource_id = CharField(null=True) rule_configuration = BinaryJSONField(null=True) + urls = BinaryJSONField(null=True) # add a potential parent_job batch_id = IntegerField(null=True) diff --git a/tests/integrationtests/test_parallel_ephemeral_processing.py b/tests/integrationtests/test_parallel_ephemeral_processing.py index a03ae76..4f1e170 100644 --- a/tests/integrationtests/test_parallel_ephemeral_processing.py +++ b/tests/integrationtests/test_parallel_ephemeral_processing.py @@ -26,6 +26,7 @@ import pytest +from flask.json import loads as json_loads from ..test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX @@ -147,3 +148,19 @@ def test_post_parallel_ephemeral_processing(self): resp["resource_response"].items() if ac_resp["process_results"] != {}] assert "stats" in process_results[0] + # Test request of one job of the batch + batch_id = resp["batch_id"] + job_id = resp["id"][0] + url = f"{URL_PREFIX}/resources/{self.user_id}/batches/{batch_id}/" \ + f"jobs/{job_id}" + rv2 = self.server.get(url, headers=self.user_auth_header) + resp2 = json_loads(rv2.data) + assert resp2["batch_id"] == batch_id, "wrong batch ID in job response" + assert resp2["id"] == int(job_id), "wrong job ID in job response" + assert "resource_response" in resp2, \ + "resource_response not in job response" + assert "urls" in resp2["resource_response"], "urls not in job response" + assert "status" in resp2["resource_response"]["urls"], \ + "status url not in job response" + assert "resource_id-" in resp2["resource_response"]["urls"][ + "status"], "resource_id not in job response" diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 4813202..bb70756 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -26,6 +26,7 @@ import time from flask.json import loads as json_loads +from urllib.parse import urlsplit from werkzeug.datastructures import Headers from actinia_core.testsuite import ActiniaTestCaseBase, URL_PREFIX @@ -140,13 +141,11 @@ def create_user(cls, name="guest", role="guest", def waitAsyncBatchJob(self, rv, headers, http_status=200, status="SUCCESS", message_check=None): resp_data = json_loads(rv.data) - batchid = resp_data["batch_id"] + url = urlsplit(resp_data["urls"]["status"]).path while True: - rv = self.server.get( - URL_PREFIX + "/processing_parallel/batchjobs/%s" % (batchid), - headers=headers - ) + rv = self.server.get(url, headers=headers) + resp_data = json_loads(rv.data) resp_data = json_loads(rv.data) if (resp_data["status"] == "SUCCESS" or resp_data["status"] == "ERROR" From 98dea57cf681d50f9abc0496f79d0d0c607ac7c9 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 21 Jul 2022 12:59:03 +0200 Subject: [PATCH 46/47] remove wsgi file --- src/actinia_parallel_plugin/wsgi.py | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/actinia_parallel_plugin/wsgi.py diff --git a/src/actinia_parallel_plugin/wsgi.py b/src/actinia_parallel_plugin/wsgi.py deleted file mode 100644 index 2f4358b..0000000 --- a/src/actinia_parallel_plugin/wsgi.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2022 mundialis GmbH & Co. KG - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -__license__ = "GPLv3" -__author__ = "Anika Weinmann" -__copyright__ = "Copyright 2022 mundialis GmbH & Co. KG" -__maintainer__ = "mundialis GmbH % Co. KG" - -from actinia_parallel_plugin.main import app as application From 1c2bee92a089547758e64b0065308bb4bfcb97b5 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 21 Jul 2022 13:15:50 +0200 Subject: [PATCH 47/47] CT review --- src/actinia_parallel_plugin/core/parallel_processing_job.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/actinia_parallel_plugin/core/parallel_processing_job.py b/src/actinia_parallel_plugin/core/parallel_processing_job.py index dac68c6..f3d34f6 100644 --- a/src/actinia_parallel_plugin/core/parallel_processing_job.py +++ b/src/actinia_parallel_plugin/core/parallel_processing_job.py @@ -28,9 +28,7 @@ from actinia_core.core.common.redis_interface import enqueue_job -from actinia_parallel_plugin.core.jobtable import ( - getJobById, -) +from actinia_parallel_plugin.core.jobtable import getJobById from actinia_parallel_plugin.core.jobs import updateJob from actinia_parallel_plugin.resources.logging import log from actinia_parallel_plugin.core.parallel_resource_base import \