feature: 订阅支持按操作系统等主机属性进行范围筛选(closed TencentBlueKing#1452)

apps/backend/subscription/serializers.py

@@ -19,6 +19,7 @@
 from apps.node_man import constants, models, tools
 from apps.node_man.models import ProcessStatus
 from apps.node_man.serializers import policy
+from apps.node_man.serializers.base import SubScopeInstSelectorSerializer
 from apps.utils import basic
 
 
@@ -27,7 +28,7 @@ class GatewaySerializer(serializers.Serializer):
     bk_app_code = serializers.CharField()
 
 
-class ScopeSerializer(serializers.Serializer):
+class ScopeSerializer(SubScopeInstSelectorSerializer):
     bk_biz_id = serializers.IntegerField(required=False, default=None)
     # TODO: 是否取消掉这个范围内的scope
     bk_biz_scope = serializers.ListField(required=False)
@@ -124,7 +125,7 @@ class GetSubscriptionSerializer(GatewaySerializer):
 
 
 class UpdateSubscriptionSerializer(GatewaySerializer):
-    class UpdateScopeSerializer(serializers.Serializer):
+    class UpdateScopeSerializer(SubScopeInstSelectorSerializer):
         node_type = serializers.ChoiceField(choices=models.Subscription.NODE_TYPE_CHOICES)
         nodes = serializers.ListField()
         bk_biz_id = serializers.IntegerField(required=False, default=None)
@@ -157,7 +158,7 @@ class SwitchSubscriptionSerializer(GatewaySerializer):
 
 
 class RunSubscriptionSerializer(GatewaySerializer):
-    class RunScopeSerializer(serializers.Serializer):
+    class RunScopeSerializer(SubScopeInstSelectorSerializer):
         node_type = serializers.ChoiceField(choices=models.Subscription.NODE_TYPE_CHOICES, label="节点类型")
         nodes = serializers.ListField(child=serializers.DictField(), label="拓扑节点列表")
 

apps/backend/subscription/tools.py

@@ -41,6 +41,7 @@
 from apps.utils.batch_request import batch_request, request_multi_thread
 from apps.utils.cache import func_cache_decorator
 from apps.utils.time_handler import strftime_local
+from apps.core.ipchooser.tools.base import HostQuerySqlHelper
 
 logger = logging.getLogger("app")
 
@@ -680,6 +681,7 @@ def wrapper(scope: Dict[str, Union[Dict, Any]], *args, **kwargs) -> Dict[str, Di
                     "object_type": scope["object_type"],
                     "node_type": scope["node_type"],
                     "nodes": list(nodes),
+                    "instance_selector": scope.get("instance_selector")
                 },
                 **kwargs,
             }
@@ -727,6 +729,11 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
         "host|instance|host|yyyy": {...},
     }
     """
+    instance_selector = scope.get("instance_selector")
+    # 不进行主机筛选时传入 None，传入空列表则识别为全部过滤
+    if instance_selector == []:
+        return {}
+
     instances = []
     bk_biz_id = scope["bk_biz_id"]
     if bk_biz_id:
@@ -808,13 +815,35 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
         "object_type": scope["object_type"],
         "node_type": models.Subscription.NodeType.INSTANCE,
     }
+
+    bk_host_ids = []
+
     for instance in instances:
-        if data["object_type"] == models.Subscription.ObjectType.HOST:
-            data.update(instance["host"])
-        else:
-            data.update(instance["service"])
+        is_host = data["object_type"] == models.Subscription.ObjectType.HOST
+        instance_data = instance["host"] if is_host else instance["service"]
+
+        data.update(instance_data)
+        bk_host_ids.append(instance_data.get("bk_host_id"))
         instances_dict[create_node_id(data)] = instance
 
+    # 对 instances 进行二次过滤
+    if instance_selector and bk_host_ids:
+        instance_selector_host_ids = HostQuerySqlHelper.multiple_cond_sql(
+            params={"bk_host_id": bk_host_ids, "conditions": instance_selector},
+            biz_scope=[bk_biz_id],
+            return_all_node_type=True
+        ).values_list("bk_host_id", flat=True)
+
+        selector_instances_dict = {}
+        for node_id, instance in instances_dict.items():
+            is_host = data["object_type"] == models.Subscription.ObjectType.HOST
+            instance_data = instance["host"] if is_host else instance["service"]
+
+            if instance_data["bk_host_id"] in instance_selector_host_ids:
+                selector_instances_dict[node_id] = instance if is_host else instance["service"]
+
+        return selector_instances_dict
+
     return instances_dict
 
 

apps/backend/subscription/views.py

@@ -84,6 +84,7 @@ def create_subscription(self, request):
                 object_type=scope["object_type"],
                 node_type=scope["node_type"],
                 nodes=scope["nodes"],
+                instance_selector=scope.get("instance_selector"),
                 target_hosts=params.get("target_hosts"),
                 from_system=params["bk_app_code"] or "blueking",
                 enable=enable,
@@ -202,6 +203,9 @@ def update_subscription(self, request):
             subscription.node_type = scope["node_type"]
             subscription.nodes = scope["nodes"]
             subscription.bk_biz_id = scope.get("bk_biz_id")
+            # 避免空列表误判
+            if scope.get("instance_selector") is not None:
+                subscription.instance_selector = scope["instance_selector"]
             # 策略部署新增
             subscription.plugin_name = params.get("plugin_name")
             subscription.bk_biz_scope = params.get("bk_biz_scope")

apps/backend/tests/subscription/test_tools.py

@@ -22,6 +22,7 @@
     CmdbClient,
     list_biz_hosts_without_info_client,
 )
+from apps.node_man import models, constants
 
 # 全局使用的mock
 run_task = mock.patch("apps.backend.subscription.tasks.run_subscription_task").start()
@@ -57,6 +58,25 @@ def setUp(self):
         self.get_process_by_biz_id_client.start()
         self.batch_request_client.start()
 
+        models.Host.objects.create(
+            bk_host_id=1,
+            bk_biz_id=2,
+            bk_cloud_id=0,
+            inner_ip="127.0.0.1",
+            outer_ip=None,
+            login_ip="127.0.0.1",
+            data_ip="127.0.0.1",
+            os_type="LINUX",
+            node_type="AGENT",
+            ap_id=1,
+        )
+        models.ProcessStatus.objects.create(
+            bk_host_id=1,
+            name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME,
+            proc_type=constants.ProcType.AGENT,
+            source_type=models.ProcessStatus.SourceType.DEFAULT,
+        )
+
     def tearDown(self):
         self.tools_client.stop()
         self.commons_client.stop()
@@ -155,3 +175,31 @@ def test_get_service_instance_scope(self):
             instance = instances[instance_id]
             self.assertEqual(instance["service"]["id"], 10)
             self.assertSetEqual({"process", "scope", "host", "service"}, set(instance.keys()))
+
+    def test_get_instance_selector_scope(self):
+        instances = get_instances_by_scope(
+            {
+                "bk_biz_id": 2,
+                "object_type": "HOST",
+                "node_type": "INSTANCE",
+                "instance_selector": [{"key": "os_type", "value": ["WINDOWS"]}],
+                "nodes": [
+                    {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0},
+                ],
+            }
+        )
+        self.assertEqual(len(list(instances.keys())), 0)
+
+    def test_get_empty_list_instance_selector_scope(self):
+        instances = get_instances_by_scope(
+            {
+                "bk_biz_id": 2,
+                "object_type": "HOST",
+                "node_type": "INSTANCE",
+                "instance_selector": [],
+                "nodes": [
+                    {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0},
+                ],
+            }
+        )
+        self.assertEqual(len(list(instances.keys())), 0)

apps/node_man/handlers/policy.py

@@ -395,6 +395,7 @@ def migrate_preview(cls, query_params: Dict[str, Any]) -> List[Dict[str, Any]]:
                 object_type=scope["object_type"],
                 node_type=scope["node_type"],
                 nodes=scope["nodes"],
+                instance_selector=scope.get("instance_selector"),
                 target_hosts=query_params.get("target_hosts"),
                 # SaaS侧均为主程序部署
                 is_main=True,

apps/node_man/migrations/0072_subscription_instance_selector.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.4 on 2023-08-04 02:19
+
+from django.db import migrations
+import django_mysql.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('node_man', '0071_update_ap_gse_version_to_v2'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscription',
+            name='instance_selector',
+            field=django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name='订阅任务范围主机属性筛选'),
+        ),
+    ]

apps/node_man/models.py

@@ -1765,6 +1765,7 @@ class CategoryType(object):
     object_type = models.CharField(_("对象类型"), max_length=20, choices=OBJECT_TYPE_CHOICES, db_index=True)
     node_type = models.CharField(_("节点类型"), max_length=20, choices=NODE_TYPE_CHOICES, db_index=True)
     nodes = JSONField(_("节点"), default=list)
+    instance_selector = JSONField(_("订阅任务范围主机属性筛选"), null=True, blank=True)
     target_hosts = JSONField(_("下发的目标机器"), default=None, null=True)
     from_system = models.CharField(_("所属系统"), max_length=30)
     update_time = models.DateTimeField(_("更新时间"), auto_now=True, db_index=True)
@@ -1808,6 +1809,7 @@ def scope(self):
             "node_type": self.node_type,
             "nodes": self.nodes,
             "need_register": need_register,
+            "instance_selector": self.instance_selector,
         }
 
     @classmethod

apps/node_man/serializers/base.py

@@ -15,6 +15,15 @@
 from apps.node_man import constants, exceptions, models
 
 
+# 放在后台会导致循坏导入
+class SubScopeInstSelectorSerializer(serializers.Serializer):
+    instance_selector = serializers.ListField(
+        child=serializers.DictField(),
+        required=False,
+        label="实例筛选器"
+    )
+
+
 # 安装插件配置
 class StepSerializer(serializers.Serializer):
     class SettingSerializer(serializers.Serializer):
@@ -46,7 +55,7 @@ def validate(self, data):
 
 
 # 策略范围
-class ScopeSerializer(serializers.Serializer):
+class ScopeSerializer(SubScopeInstSelectorSerializer):
     class NodeSerializer(serializers.Serializer):
         bk_biz_id = serializers.IntegerField(label="业务ID")
         bk_inst_id = serializers.IntegerField(required=False, label="实例ID")

docs/apidoc/create_subscription.md

@@ -34,12 +34,13 @@
 
 | 字段            | 类型        | 必选  | 描述                                                                                  |
 | ------------- | --------- | --- | ----------------------------------------------------------------------------------- |
-| bk_biz_id     | int       | 否   | 蓝鲸业务ID                                                                              |
-| bk_biz_scope  | int array | 否   | 蓝鲸业务ID列表                                                                            |
-| node_type     | string    | 是   | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 |
-| object_type   | string    | 是   | 对象类型，1：HOST，主机类型  2：SERVICE，服务类型                                                    |
-| need_register | bool      | 否   | 是否需要注册到CMDB，false是不注册，true是注册。默认为不注册                                                |
-| nodes         | objects   | 是   | 节点列表，见nodes定义                                                                       |
+| bk_biz_id         | int       | 否   | 蓝鲸业务ID                                                                              |
+| bk_biz_scope      | int array | 否   | 蓝鲸业务ID列表                                                                            |
+| node_type         | string    | 是   | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 |
+| object_type       | string    | 是   | 对象类型，1：HOST，主机类型  2：SERVICE，服务类型                                                    |
+| need_register     | bool      | 否   | 是否需要注册到CMDB，false是不注册，true是注册。默认为不注册                                                |
+| nodes             | objects   | 是   | 节点列表，见nodes定义        
+| instance_selector | objects   | 否   | 主机属性筛选列表                                                                 |
 
 ##### config
 
@@ -119,6 +120,13 @@ instance_info
 | enable_compression             | bool   | 否   | 数据压缩开关，默认是关闭                                              |
 | data_path                      | string | 否   | 数据文件路径                                                  |
 
+###### instance_selector
+
+| 字段                  | 类型     | 必选  | 描述                      |
+| ------------------- | ------ | --- | ----------------------- |
+| key   | string    | 否   | 主机属性                   |
+| value | string    | 否   | 主机属性值列表                   |
+
 ###### job_type
 
 Agent
@@ -184,6 +192,7 @@ Plugin
     "bk_token": "xxx",
     "run_immediately": true,
     "scope": {
+        "instance_selector": [{"key": "os_type", "value": ["LINUX"]}],
         "bk_biz_id": 2,
         "object_type": "SERVICE",
         "node_type": "TOPO",

docs/apidoc/update_subscription.md

@@ -31,9 +31,10 @@
 
 | 字段        | 类型            | 必选  | 描述                                                                                  |
 | --------- | ------------- | --- | ----------------------------------------------------------------------------------- |
-| bk_biz_id | int           | 否   | 蓝鲸业务ID                                                                              |
-| node_type | string        | 是   | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 |
-| nodes     | objects       | 是   | 节点列表，见nodes定义                                                                       |
+| bk_biz_id         | int           | 否   | 蓝鲸业务ID                                                                              |
+| node_type         | string        | 是   | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 |
+| nodes             | objects       | 是   | 节点列表，见nodes定义                                                |
+| instance_selector | objects   | 否   | 主机属性筛选列表                                  |
 
 ##### config
 
@@ -113,6 +114,13 @@ instance_info
 | data_path                      | string | 否   | 数据文件路径                                                          |
 | enable_compression             | bool   | 否   | 数据压缩开关                                                          |
 
+###### instance_selector
+
+| 字段                  | 类型     | 必选  | 描述                      |
+| ------------------- | ------ | --- | ----------------------- |
+| key   | string    | 否   | 主机属性                   |
+| value | string    | 否   | 主机属性值                   |
+
 ###### job_type
 
 Agent
@@ -180,6 +188,7 @@ Plugin
     "run_immediately": true,
     "subscription_id": 1,
     "scope": {
+        "instance_selector": [{"key": "os_type", "value": ["LINUX"]}],
         "bk_biz_id": 2,
         "object_type": "SERVICE",
         "node_type": "TOPO",