From f849bd693970ae4a0676a57959d9068dcbb9d6b4 Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Tue, 14 Jun 2022 15:21:01 +1000 Subject: [PATCH 1/5] [SDESK-6441] Ability to register new routing rule handlers --- apps/rules/routing_rules.py | 154 ++++-------------------------- apps/rules/rule_handlers.py | 185 ++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 133 deletions(-) create mode 100644 apps/rules/rule_handlers.py diff --git a/apps/rules/routing_rules.py b/apps/rules/routing_rules.py index ab6199413d..27a835accb 100644 --- a/apps/rules/routing_rules.py +++ b/apps/rules/routing_rules.py @@ -16,15 +16,15 @@ from enum import Enum from datetime import datetime, timedelta -from superdesk import get_resource_service from superdesk.resource import Resource from superdesk.services import BaseService from superdesk.errors import SuperdeskApiError from eve.utils import config -from superdesk.metadata.item import CONTENT_STATE from superdesk.utc import set_time from flask_babel import _ +from .rule_handlers import get_routing_rule_handler + logger = logging.getLogger(__name__) @@ -78,6 +78,7 @@ class RoutingRuleSchemeResource(Resource): "type": "dict", "schema": { "name": {"type": "string"}, + "handler": {"type": "string", "required": True, "nullable": False, "empty": False}, "filter": Resource.rel("content_filters", nullable=True), "actions": { "type": "dict", @@ -108,6 +109,12 @@ class RoutingRuleSchemeResource(Resource): }, "exit": {"type": "boolean"}, "preserve_desk": {"type": "boolean"}, + "extra": { + "type": "dict", + "nullable": True, + "schema": {}, + "allow_unknown": True, + }, }, }, "schedule": { @@ -203,28 +210,20 @@ def apply_routing_scheme(self, ingest_item, provider, routing_scheme): "Applying rule. Item: %s . Routing Scheme: %s. Rule Name %s." % (ingest_item.get("guid"), routing_scheme.get("name"), rule.get("name")) ) - if filters_service.does_match(content_filter, ingest_item): + + rule_handler = get_routing_rule_handler(rule) + if not rule_handler.can_handle(rule, ingest_item, routing_scheme): + logger.info( + "Routing rule %s of Routing Scheme %s for Provider %s does not support item %s" + % (rule.get("name"), routing_scheme.get("name"), provider.get("name"), ingest_item[config.ID_FIELD]) + ) + elif filters_service.does_match(content_filter, ingest_item): logger.info( "Filter matched. Item: %s. Routing Scheme: %s. Rule Name %s." % (ingest_item.get("guid"), routing_scheme.get("name"), rule.get("name")) ) - if rule.get("actions", {}).get("preserve_desk", False) and ingest_item.get("task", {}).get("desk"): - desk = get_resource_service("desks").find_one(req=None, _id=ingest_item["task"]["desk"]) - if ingest_item.get("task", {}).get("stage"): - stage_id = ingest_item["task"]["stage"] - else: - stage_id = desk["incoming_stage"] - self.__fetch(ingest_item, [{"desk": desk[config.ID_FIELD], "stage": stage_id}], rule) - fetch_actions = [ - f - for f in rule.get("actions", {}).get("fetch", []) - if f.get("desk") != ingest_item["task"]["desk"] - ] - else: - fetch_actions = rule.get("actions", {}).get("fetch", []) - self.__fetch(ingest_item, fetch_actions, rule) - self.__publish(ingest_item, rule.get("actions", {}).get("publish", []), rule) + rule_handler.apply_rule(rule, ingest_item, routing_scheme) if rule.get("actions", {}).get("exit", False): logger.info( "Exiting routing scheme. Item: %s . Routing Scheme: %s. " @@ -281,7 +280,9 @@ def _validate_routing_scheme(self, routing_scheme): raise SuperdeskApiError.badRequestError(message=_("A Routing Scheme must have at least one Rule")) for routing_rule in routing_rules: invalid_fields = [ - field for field in routing_rule.keys() if field not in ("name", "filter", "actions", "schedule") + field + for field in routing_rule.keys() + if field not in ("name", "handler", "filter", "actions", "schedule") ] if invalid_fields: @@ -410,116 +411,3 @@ def _get_scheduled_routing_rules(self, rules, current_dt_utc): scheduled_rules.append(rule) return scheduled_rules - - def __fetch(self, ingest_item, destinations, rule): - """Fetch to item to the destinations - - :param item: item to be fetched - :param destinations: list of desk and stage - """ - archive_items = [] - for destination in destinations: - try: - logger.info("Fetching item %s to desk %s" % (ingest_item.get("guid"), destination)) - target = self.__getTarget(destination) - item_id = get_resource_service("fetch").fetch( - [ - { - config.ID_FIELD: ingest_item[config.ID_FIELD], - "desk": str(destination.get("desk")), - "stage": str(destination.get("stage")), - "state": CONTENT_STATE.ROUTED, - "macro": destination.get("macro", None), - "target": target, - }, - ], - macro_kwargs={ - "rule": rule, - }, - )[0] - archive_items.append(item_id) - logger.info("Fetched item %s to desk %s" % (ingest_item.get("guid"), destination)) - except Exception: - logger.exception("Failed to fetch item %s to desk %s" % (ingest_item.get("guid"), destination)) - - return archive_items - - def __getTarget(self, destination): - """Get the target for destination - - :param dict destination: routing destination - :return dict: returns target information - """ - target = {} - if destination.get("target_subscribers"): - target["target_subscribers"] = destination.get("target_subscribers") - - if destination.get("target_types"): - target["target_types"] = destination.get("target_types") - - return target - - def __publish(self, ingest_item, destinations, rule): - """Fetches the item to the desk and then publishes the item. - - :param item: item to be published - :param destinations: list of desk and stage - """ - guid = ingest_item.get("guid") - items_to_publish = self.__fetch(ingest_item, destinations, rule) - for item in items_to_publish: - try: - archive_item = get_resource_service("archive").find_one(req=None, _id=item) - if archive_item.get("auto_publish") is False: - logger.info("Stop auto publishing of item %s", guid) - continue - logger.info("Publishing item %s", guid) - self._set_default_values(archive_item) - get_resource_service("archive_publish").patch(item, {"auto_publish": True}) - logger.info("Published item %s", guid) - except Exception: - logger.exception("Failed to publish item %s.", guid) - - def _set_default_values(self, archive_item): - """Assigns the default values to the item that about to be auto published""" - default_categories = self._get_categories(config.DEFAULT_CATEGORY_QCODES_FOR_AUTO_PUBLISHED_ARTICLES) - default_values = self._assign_default_values(archive_item, default_categories) - get_resource_service("archive").patch(archive_item["_id"], default_values) - - def _assign_default_values(self, archive_item, default_categories): - """Assigns the default values to the item that about to be auto published""" - - default_values = {} - default_values["headline"] = archive_item.get("headline") or " " - - if archive_item.get("anpa_category"): - default_values["anpa_category"] = archive_item.get("anpa_category") - else: - default_values["anpa_category"] = default_categories - - default_values["slugline"] = archive_item.get("slugline") or " " - default_values["body_html"] = archive_item.get("body_html") or "

" - return default_values - - def _get_categories(self, qcodes): - """Returns list of categories for a given comma separated qcodes""" - - if not qcodes: - return - - qcode_list = qcodes.split(",") - selected_categories = None - categories = superdesk.get_resource_service("vocabularies").find_one(req=None, _id="categories") - - if categories and len(qcode_list) > 0: - selected_categories = [] - for qcode in qcode_list: - selected_categories.extend( - [ - {"qcode": qcode, "name": c.get("name", "")} - for c in categories["items"] - if c["is_active"] is True and qcode.lower() == c["qcode"].lower() - ] - ) - - return selected_categories diff --git a/apps/rules/rule_handlers.py b/apps/rules/rule_handlers.py new file mode 100644 index 0000000000..6e6e0ca2c5 --- /dev/null +++ b/apps/rules/rule_handlers.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2022 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +from typing import Dict +import logging + +from eve.utils import config +from superdesk import get_resource_service +from superdesk.metadata.item import CONTENT_STATE, ITEM_TYPE, CONTENT_TYPE, MEDIA_TYPES +from superdesk.errors import AlreadyExistsError + +logger = logging.getLogger(__name__) + + +class RoutingRuleHandler: + NAME: str + + def can_handle(self, rule, ingest_item, routing_scheme) -> bool: + raise NotImplementedError() + + def apply_rule(self, rule, ingest_item, routing_scheme): + raise NotImplementedError() + + +registered_routing_rule_handlers: Dict[str, RoutingRuleHandler] = {} + + +def register_routing_rule_handler(routing_handler: RoutingRuleHandler): + if registered_routing_rule_handlers.get(routing_handler.NAME): + raise AlreadyExistsError(f"Ingest Publisher: {routing_handler.NAME} already registered") + + registered_routing_rule_handlers[routing_handler.NAME] = routing_handler + + +def get_routing_rule_handler(rule) -> RoutingRuleHandler: + return registered_routing_rule_handlers[rule.get("handler", DeskFetchPublishRoutingRuleHandler.NAME)] + + +class DeskFetchPublishRoutingRuleHandler(RoutingRuleHandler): + NAME = "desk_fetch_publish" + + def can_handle(self, rule, ingest_item, routing_scheme): + return ingest_item.get(ITEM_TYPE) in ( + MEDIA_TYPES + (CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED, CONTENT_TYPE.COMPOSITE) + ) + + def apply_rule(self, rule, ingest_item, routing_scheme): + if rule.get("actions", {}).get("preserve_desk", False) and ingest_item.get("task", {}).get("desk"): + desk = get_resource_service("desks").find_one(req=None, _id=ingest_item["task"]["desk"]) + if ingest_item.get("task", {}).get("stage"): + stage_id = ingest_item["task"]["stage"] + else: + stage_id = desk["incoming_stage"] + self.__fetch(ingest_item, [{"desk": desk[config.ID_FIELD], "stage": stage_id}], rule) + fetch_actions = [ + f for f in rule.get("actions", {}).get("fetch", []) if f.get("desk") != ingest_item["task"]["desk"] + ] + else: + fetch_actions = rule.get("actions", {}).get("fetch", []) + + self.__fetch(ingest_item, fetch_actions, rule) + self.__publish(ingest_item, rule.get("actions", {}).get("publish", []), rule) + + def __fetch(self, ingest_item, destinations, rule): + """Fetch to item to the destinations + + :param item: item to be fetched + :param destinations: list of desk and stage + """ + archive_items = [] + for destination in destinations: + try: + logger.info("Fetching item %s to desk %s" % (ingest_item.get("guid"), destination)) + target = self.__get_target(destination) + item_id = get_resource_service("fetch").fetch( + [ + { + config.ID_FIELD: ingest_item[config.ID_FIELD], + "desk": str(destination.get("desk")), + "stage": str(destination.get("stage")), + "state": CONTENT_STATE.ROUTED, + "macro": destination.get("macro", None), + "target": target, + }, + ], + macro_kwargs={ + "rule": rule, + }, + )[0] + archive_items.append(item_id) + logger.info("Fetched item %s to desk %s" % (ingest_item.get("guid"), destination)) + except Exception: + logger.exception("Failed to fetch item %s to desk %s" % (ingest_item.get("guid"), destination)) + + return archive_items + + def __publish(self, ingest_item, destinations, rule): + """Fetches the item to the desk and then publishes the item. + + :param item: item to be published + :param destinations: list of desk and stage + """ + guid = ingest_item.get("guid") + items_to_publish = self.__fetch(ingest_item, destinations, rule) + for item in items_to_publish: + try: + archive_item = get_resource_service("archive").find_one(req=None, _id=item) + if archive_item.get("auto_publish") is False: + logger.info("Stop auto publishing of item %s", guid) + continue + logger.info("Publishing item %s", guid) + self._set_default_values(archive_item) + get_resource_service("archive_publish").patch(item, {"auto_publish": True}) + logger.info("Published item %s", guid) + except Exception: + logger.exception("Failed to publish item %s.", guid) + + def __get_target(self, destination): + """Get the target for destination + + :param dict destination: routing destination + :return dict: returns target information + """ + target = {} + if destination.get("target_subscribers"): + target["target_subscribers"] = destination.get("target_subscribers") + + if destination.get("target_types"): + target["target_types"] = destination.get("target_types") + + return target + + def _set_default_values(self, archive_item): + """Assigns the default values to the item that about to be auto published""" + default_categories = self._get_categories(config.DEFAULT_CATEGORY_QCODES_FOR_AUTO_PUBLISHED_ARTICLES) + default_values = self._assign_default_values(archive_item, default_categories) + get_resource_service("archive").patch(archive_item["_id"], default_values) + + def _assign_default_values(self, archive_item, default_categories): + """Assigns the default values to the item that about to be auto published""" + + default_values = {} + default_values["headline"] = archive_item.get("headline") or " " + + if archive_item.get("anpa_category"): + default_values["anpa_category"] = archive_item.get("anpa_category") + else: + default_values["anpa_category"] = default_categories + + default_values["slugline"] = archive_item.get("slugline") or " " + default_values["body_html"] = archive_item.get("body_html") or "

" + return default_values + + def _get_categories(self, qcodes): + """Returns list of categories for a given comma separated qcodes""" + + if not qcodes: + return + + qcode_list = qcodes.split(",") + selected_categories = None + categories = get_resource_service("vocabularies").find_one(req=None, _id="categories") + + if categories and len(qcode_list) > 0: + selected_categories = [] + for qcode in qcode_list: + selected_categories.extend( + [ + {"qcode": qcode, "name": c.get("name", "")} + for c in categories["items"] + if c["is_active"] is True and qcode.lower() == c["qcode"].lower() + ] + ) + + return selected_categories + + +register_routing_rule_handler(DeskFetchPublishRoutingRuleHandler()) From 43ab620dc0b59b278d537d50c177d4b7202bfa87 Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Tue, 14 Jun 2022 15:22:12 +1000 Subject: [PATCH 2/5] fix tests --- features/archive_history.feature | 1 + features/auto_routing.feature | 14 ++++++++++++++ features/content_filter.feature | 1 + features/routing_rules.feature | 24 ++++++++++++++++++++++++ features/stages.feature | 7 ++++--- tests/rules/set_default_values_test.py | 6 +++--- 6 files changed, 47 insertions(+), 6 deletions(-) diff --git a/features/archive_history.feature b/features/archive_history.feature index e3c8b5bde1..679f2fae22 100644 --- a/features/archive_history.feature +++ b/features/archive_history.feature @@ -752,6 +752,7 @@ Feature: Archive history "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": null, "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], diff --git a/features/auto_routing.feature b/features/auto_routing.feature index 5001d0cf94..c99c417d2b 100644 --- a/features/auto_routing.feature +++ b/features/auto_routing.feature @@ -60,6 +60,7 @@ Feature: Auto Routing "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [ @@ -181,6 +182,7 @@ Feature: Auto Routing "rules": [ { "name": "Syria Rule", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], @@ -251,6 +253,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], @@ -367,6 +370,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], @@ -465,6 +469,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], @@ -539,6 +544,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "preserve_desk": true, @@ -609,6 +615,7 @@ Feature: Auto Routing "rules": [ { "name": "Politics Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#", "macro": "update_fields"}], @@ -705,6 +712,7 @@ Feature: Auto Routing "rules": [ { "name": "Syria Rule", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], @@ -780,6 +788,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [], @@ -852,6 +861,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [], @@ -958,6 +968,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [], @@ -1076,6 +1087,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [], @@ -1164,6 +1176,7 @@ Feature: Auto Routing "rules": [ { "name": "publish", + "handler": "desk_fetch_publish", "actions": { "publish": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}], "exit": false @@ -1249,6 +1262,7 @@ Feature: Auto Routing "rules": [ { "name": "Finance Rule 1", + "handler": "desk_fetch_publish", "filter": "1234567890abcd1234567890", "actions": { "fetch": [], diff --git a/features/content_filter.feature b/features/content_filter.feature index 5fe21ba1a5..d5e80c8404 100644 --- a/features/content_filter.feature +++ b/features/content_filter.feature @@ -242,6 +242,7 @@ Feature: Content Filter "name": "routing scheme 1", "rules": [{ "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#content_filters._id#", "actions": { "fetch": [] diff --git a/features/routing_rules.feature b/features/routing_rules.feature index b42a570b14..c13ef2f18f 100644 --- a/features/routing_rules.feature +++ b/features/routing_rules.feature @@ -52,6 +52,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [ @@ -76,6 +77,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#", "macro": "transform"}] @@ -94,6 +96,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "publish": [ @@ -116,6 +119,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "publish": [ @@ -171,6 +175,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -189,6 +194,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -274,6 +280,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Content Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_1_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -281,6 +288,7 @@ Feature: Routing Scheme and Routing Rules }, { "name": "Content Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_2_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -308,6 +316,7 @@ Feature: Routing Scheme and Routing Rules "name": "routing rule scheme 1", "rules": [{ "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": null, "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -351,6 +360,7 @@ Feature: Routing Scheme and Routing Rules "name": "routing rule scheme 1", "rules": [{ "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": {} }] @@ -400,6 +410,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -422,6 +433,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -446,6 +458,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -470,6 +483,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -494,6 +508,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -519,6 +534,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -543,6 +559,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -574,6 +591,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": null, "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -646,6 +664,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_1_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -662,6 +681,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_1_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#", "macro": "transform"}] @@ -669,6 +689,7 @@ Feature: Routing Scheme and Routing Rules }, { "name": "Non-Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_2_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -717,6 +738,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -746,6 +768,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": null, "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] @@ -778,6 +801,7 @@ Feature: Routing Scheme and Routing Rules "rules": [ { "name": "Sports Rule", + "handler": "desk_fetch_publish", "filter": null, "actions": { "fetch": [{"desk": "#desks._id#", "stage": "#desks.incoming_stage#"}] diff --git a/features/stages.feature b/features/stages.feature index f763d9b658..72f4e76144 100644 --- a/features/stages.feature +++ b/features/stages.feature @@ -213,9 +213,10 @@ Feature: Stages """ And we post to "/routing_schemes" """ - [{"name": "routing rule scheme 1", "rules": [{"name": "Sports Rule", "filter": "#FILTER_ID#", - "actions": {"fetch": [{"desk": "#desks._id#", "stage": "#stages._id#", "macro": "transform"}]}}]} - ] + [{"name": "routing rule scheme 1", "rules": [{ + "name": "Sports Rule", "handler": "desk_fetch_publish", "filter": "#FILTER_ID#", + "actions": {"fetch": [{"desk": "#desks._id#", "stage": "#stages._id#", "macro": "transform"}]} + }]}] """ And we delete "/stages/#stages._id#" Then we get error 412 diff --git a/tests/rules/set_default_values_test.py b/tests/rules/set_default_values_test.py index 55b55b08df..2fc60b9977 100644 --- a/tests/rules/set_default_values_test.py +++ b/tests/rules/set_default_values_test.py @@ -1,11 +1,11 @@ from superdesk.tests import TestCase -from apps.rules.routing_rules import RoutingRuleSchemeService +from apps.rules.rule_handlers import DeskFetchPublishRoutingRuleHandler class SetDefaultValuesTestCase(TestCase): def test_setting_default_values(self): - instance = RoutingRuleSchemeService() + instance = DeskFetchPublishRoutingRuleHandler() category = [{"qcode": "a", "name": "Australian General News"}] result = instance._assign_default_values({"anpa_category": category}, None) self.assertEqual(result["anpa_category"], [{"qcode": "a", "name": "Australian General News"}]) @@ -29,7 +29,7 @@ def test_getting_selected_categories(self): } ] - instance = RoutingRuleSchemeService() + instance = DeskFetchPublishRoutingRuleHandler() with self.app.app_context(): self.app.data.insert("vocabularies", vocabularies) From a60053d33de1dac5bc4f2c993f1f223a1f5463eb Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Thu, 16 Jun 2022 15:38:17 +1000 Subject: [PATCH 3/5] Define IngestRuleHandlers in backend and provide endpoint to get them --- apps/rules/__init__.py | 5 ++ apps/rules/routing_rules.py | 2 +- apps/rules/rule_handlers.py | 93 +++++++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/apps/rules/__init__.py b/apps/rules/__init__.py index 61f17052b7..d7e3c223c5 100644 --- a/apps/rules/__init__.py +++ b/apps/rules/__init__.py @@ -15,6 +15,7 @@ from .routing_rules import RoutingRuleSchemeResource, RoutingRuleSchemeService from .rule_sets import RuleSetsService, RuleSetsResource +from .rule_handlers import IngestRuleHandlersResource, IngestRuleHandlersService from superdesk import get_backend import superdesk @@ -30,6 +31,10 @@ def init_app(app) -> None: service = RoutingRuleSchemeService(endpoint_name, backend=get_backend()) RoutingRuleSchemeResource(endpoint_name, app=app, service=service) + endpoint_name = "ingest_rule_handlers" + service = IngestRuleHandlersService(endpoint_name, backend=get_backend()) + IngestRuleHandlersResource(endpoint_name, app=app, service=service) + superdesk.privilege( name="rule_sets", label=lazy_gettext("Transformation Rules Management"), diff --git a/apps/rules/routing_rules.py b/apps/rules/routing_rules.py index 27a835accb..af4a6fed4b 100644 --- a/apps/rules/routing_rules.py +++ b/apps/rules/routing_rules.py @@ -78,7 +78,7 @@ class RoutingRuleSchemeResource(Resource): "type": "dict", "schema": { "name": {"type": "string"}, - "handler": {"type": "string", "required": True, "nullable": False, "empty": False}, + "handler": {"type": "string", "nullable": False, "default": "desk_fetch_publish"}, "filter": Resource.rel("content_filters", nullable=True), "actions": { "type": "dict", diff --git a/apps/rules/rule_handlers.py b/apps/rules/rule_handlers.py index 6e6e0ca2c5..c029604406 100644 --- a/apps/rules/rule_handlers.py +++ b/apps/rules/rule_handlers.py @@ -8,19 +8,25 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license -from typing import Dict +from typing import Dict, Any import logging from eve.utils import config -from superdesk import get_resource_service +from flask_babel import lazy_gettext, LazyString + +from superdesk import get_resource_service, Resource, Service from superdesk.metadata.item import CONTENT_STATE, ITEM_TYPE, CONTENT_TYPE, MEDIA_TYPES -from superdesk.errors import AlreadyExistsError +from superdesk.utils import ListCursor logger = logging.getLogger(__name__) class RoutingRuleHandler: - NAME: str + ID: str + NAME: LazyString + supported_actions: Dict[str, bool] + supported_configs: Dict[str, bool] + default_values: Dict[str, Any] def can_handle(self, rule, ingest_item, routing_scheme) -> bool: raise NotImplementedError() @@ -33,18 +39,85 @@ def apply_rule(self, rule, ingest_item, routing_scheme): def register_routing_rule_handler(routing_handler: RoutingRuleHandler): - if registered_routing_rule_handlers.get(routing_handler.NAME): - raise AlreadyExistsError(f"Ingest Publisher: {routing_handler.NAME} already registered") - - registered_routing_rule_handlers[routing_handler.NAME] = routing_handler + registered_routing_rule_handlers[routing_handler.ID] = routing_handler def get_routing_rule_handler(rule) -> RoutingRuleHandler: - return registered_routing_rule_handlers[rule.get("handler", DeskFetchPublishRoutingRuleHandler.NAME)] + return registered_routing_rule_handlers[rule.get("handler", DeskFetchPublishRoutingRuleHandler.ID)] + + +class IngestRuleHandlersResource(Resource): + item_methods = [] + resource_methods = ["GET"] + schema = { + "_id": {"type": "string"}, + "name": {"type": "string"}, + "supported_actions": { + "type": "dict", + "required": False, + "schema": {}, + "allow_unknown": True, + }, + "supported_configs": { + "type": "dict", + "required": False, + "schema": {}, + "allow_unknown": True, + }, + "default_values": { + "type": "dict", + "required": False, + "schema": {}, + "allow_unknown": True, + }, + } + + +class IngestRuleHandlersService(Service): + def get(self, req, lookup): + """Return list of available ingest rule handlers""" + + values = sorted( + [ + dict( + _id=handler.ID, + name=handler.NAME, + supported_actions=handler.supported_actions, + supported_configs=handler.supported_configs, + default_values=handler.default_values, + ) + for handler in registered_routing_rule_handlers.values() + ], + key=lambda x: x["name"].lower(), + ) + + return ListCursor(values) class DeskFetchPublishRoutingRuleHandler(RoutingRuleHandler): - NAME = "desk_fetch_publish" + ID = "desk_fetch_publish" + NAME = lazy_gettext("Desk Fetch/Publish") + supported_actions = { + "fetch_to_desk": True, + "publish_from_desk": True, + } + supported_configs = {"exit": True, "preserve_desk": True} + default_values = { + "name": "", + "handler": "desk_fetch_publish", + "filter": None, + "actions": { + "fetch": [], + "publish": [], + "exit": False, + }, + "schedule": { + "day_of_week": ["MON", "TUE", "WED", "THU", "FRI", "SAT", "SUN"], + "hour_of_day_from": None, + "hour_of_day_to": None, + "_allDay": True, + }, + } def can_handle(self, rule, ingest_item, routing_scheme): return ingest_item.get(ITEM_TYPE) in ( From d4ca792e1c5772a6e6e3eb5842dfb7cceb4a2b59 Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Thu, 16 Jun 2022 15:39:07 +1000 Subject: [PATCH 4/5] behave test for `ingest_rule_handlers` --- features/ingest_rule_handlers.feature | 85 +++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 features/ingest_rule_handlers.feature diff --git a/features/ingest_rule_handlers.feature b/features/ingest_rule_handlers.feature new file mode 100644 index 0000000000..9d19e403f0 --- /dev/null +++ b/features/ingest_rule_handlers.feature @@ -0,0 +1,85 @@ +Feature: Ingest Rule Handlers + + @auth + Scenario: Get list of available rule handlers + When we get "/ingest_rule_handlers" + Then we get list with 2 items + """ + {"_items": [ + { + "_id": "desk_fetch_publish", + "name": "Desk Fetch/Publish", + "supported_actions": { + "fetch_to_desk": true, + "publish_from_desk": true + }, + "supported_configs": { + "exit": true, + "preserve_desk": true + }, + "default_values": { + "name": "", + "filter": null, + "handler": "desk_fetch_publish", + "actions": { + "fetch": [], + "publish": [], + "exit": false + }, + "schedule": { + "day_of_week": [ + "MON", + "TUE", + "WED", + "THU", + "FRI", + "SAT", + "SUN" + ], + "hour_of_day_from": null, + "hour_of_day_to": null, + "_allDay": true + } + } + }, + { + "_id": "planning_publish", + "name": "Autopost Planning", + "supported_actions": { + "fetch_to_desk": false, + "publish_from_desk": false + }, + "supported_configs": { + "exit": true, + "preserve_desk": false + }, + "default_values": { + "name": "", + "filter": null, + "handler": "planning_publish", + "actions": { + "fetch": [], + "publish": [], + "exit": false, + "extra": { + "autopost": true + } + }, + "schedule": { + "day_of_week": [ + "MON", + "TUE", + "WED", + "THU", + "FRI", + "SAT", + "SUN" + ], + "hour_of_day_from": null, + "hour_of_day_to": null, + "_allDay": true + } + } + } + ]} + """ From 77816c3eee687db35a0359822b80a43fb649947e Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Thu, 16 Jun 2022 16:07:52 +1000 Subject: [PATCH 5/5] fix(behave): Remove Planning Autopost rule from test --- features/ingest_rule_handlers.feature | 41 +-------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/features/ingest_rule_handlers.feature b/features/ingest_rule_handlers.feature index 9d19e403f0..a3c604d70d 100644 --- a/features/ingest_rule_handlers.feature +++ b/features/ingest_rule_handlers.feature @@ -3,7 +3,7 @@ Feature: Ingest Rule Handlers @auth Scenario: Get list of available rule handlers When we get "/ingest_rule_handlers" - Then we get list with 2 items + Then we get list with 1 items """ {"_items": [ { @@ -41,45 +41,6 @@ Feature: Ingest Rule Handlers "_allDay": true } } - }, - { - "_id": "planning_publish", - "name": "Autopost Planning", - "supported_actions": { - "fetch_to_desk": false, - "publish_from_desk": false - }, - "supported_configs": { - "exit": true, - "preserve_desk": false - }, - "default_values": { - "name": "", - "filter": null, - "handler": "planning_publish", - "actions": { - "fetch": [], - "publish": [], - "exit": false, - "extra": { - "autopost": true - } - }, - "schedule": { - "day_of_week": [ - "MON", - "TUE", - "WED", - "THU", - "FRI", - "SAT", - "SUN" - ], - "hour_of_day_from": null, - "hour_of_day_to": null, - "_allDay": true - } - } } ]} """