[impv] Remove tenant define is workflow (#54)

* [impv] Remove tenant define is workflow current tenant in workflow only work when the first time user do not exist, when user change the tenant in workflow but tenant exist, it will be ignore, so we try to remove it from workflow, and in #40 we try to create both user and tenant vis cli instead of auto create
apache · Dec 19, 2022 · 1acaf89 · 1acaf89
1 parent 7722a63
commit 1acaf89
Show file tree

Hide file tree

Showing 32 changed files with 23 additions and 146 deletions.
diff --git a/UPDATING.md b/UPDATING.md
@@ -24,6 +24,11 @@ It started after version 2.0.5 released
 
 ## Main
 
+* Remove attribute tenant from pydolphinscheduler.core.workflow.workflow ([#54](https://github.com/apache/dolphinscheduler-sdk-python/pull/54))
+  and please change tenant name in ``config.yaml`` in ``PYDS_HOME``
+
+## 4.0.0
+
 * Change Task attr ``timeout`` type from int to timedelta and use timeout determine attr ``timeout_flag`` value ([#41](https://github.com/apache/dolphinscheduler-sdk-python/pull/41)) 
 * Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
 * Change class name from process definition to workflow ([#26](https://github.com/apache/dolphinscheduler-sdk-python/pull/26))

diff --git a/docs/source/concept.rst b/docs/source/concept.rst
@@ -80,11 +80,12 @@ Tenant
 ~~~~~~
 
 Tenant is the user who run task command in machine or in virtual machine. it could be assign by simple string.
+You should change the tenant value to exists tenant in your host, it config in `config.yaml` in your pydolphinscheduler
+``PYDS_HOME``, or via :doc:`CLI <cli>`
 
-.. code-block:: python
+.. code-block:: bash
 
-   # 
-   workflow = Workflow(name="workflow tenant", tenant="tenant_exists")
+   pydolphinscheduler config --set default.user.tenant <YOUR-TENANT-NAME>
 
 .. note::
 

diff --git a/docs/source/config.rst b/docs/source/config.rst
@@ -101,8 +101,6 @@ All environment variables as below, and you could modify their value via `Bash <
 +------------------+------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 |                  | ``PYDS_WORKFLOW_PROJECT``          | Default workflow project name, will use its value when workflow does not specify the attribute ``project``.         |
 +                  +------------------------------------+---------------------------------------------------------------------------------------------------------------------+
-|                  | ``PYDS_WORKFLOW_TENANT``           | Default workflow tenant, will use its value when workflow does not specify the attribute ``tenant``.                |
-+                  +------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 | Default Workflow | ``PYDS_WORKFLOW_USER``             | Default workflow user, will use its value when workflow does not specify the attribute ``user``.                    |
 +                  +------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 |                  | ``PYDS_WORKFLOW_QUEUE``            | Default workflow queue, will use its value when workflow does not specify the attribute ``queue``.                  |

diff --git a/examples/yaml_define/tutorial.yaml b/examples/yaml_define/tutorial.yaml
@@ -20,7 +20,6 @@ workflow:
   name: "tutorial"
   schedule: "0 0 0 * * ? *"
   start_time: "2021-01-01"
-  tenant: "tenant_exists"
   release_state: "offline"
   run: true
 

diff --git a/src/pydolphinscheduler/configuration.py b/src/pydolphinscheduler/configuration.py
@@ -184,6 +184,7 @@ def get_bool(val: Any) -> bool:
     "PYDS_USER_PASSWORD", configs.get("default.user.password")
 )
 USER_EMAIL = os.environ.get("PYDS_USER_EMAIL", configs.get("default.user.email"))
+USER_TENANT = os.environ.get("PYDS_USER_STATE", configs.get("default.user.tenant"))
 USER_PHONE = str(os.environ.get("PYDS_USER_PHONE", configs.get("default.user.phone")))
 USER_STATE = get_int(
     os.environ.get("PYDS_USER_STATE", configs.get("default.user.state"))
@@ -193,9 +194,6 @@ def get_bool(val: Any) -> bool:
 WORKFLOW_PROJECT = os.environ.get(
     "PYDS_WORKFLOW_PROJECT", configs.get("default.workflow.project")
 )
-WORKFLOW_TENANT = os.environ.get(
-    "PYDS_WORKFLOW_TENANT", configs.get("default.workflow.tenant")
-)
 WORKFLOW_USER = os.environ.get(
     "PYDS_WORKFLOW_USER", configs.get("default.workflow.user")
 )

diff --git a/src/pydolphinscheduler/core/default_config.yaml b/src/pydolphinscheduler/core/default_config.yaml
diff --git a/src/pydolphinscheduler/core/workflow.py b/src/pydolphinscheduler/core/workflow.py
@@ -27,7 +27,7 @@
 from pydolphinscheduler.core.resource_plugin import ResourcePlugin
 from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException
 from pydolphinscheduler.java_gateway import gateway
-from pydolphinscheduler.models import Base, Project, Tenant, User
+from pydolphinscheduler.models import Base, Project, User
 from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule
 
 
@@ -87,7 +87,6 @@ class Workflow(Base):
     _KEY_ATTR = {
         "name",
         "project",
-        "tenant",
         "release_state",
         "param",
     }
@@ -96,7 +95,6 @@ class Workflow(Base):
         "name",
         "description",
         "_project",
-        "_tenant",
         "worker_group",
         "warning_type",
         "warning_group_id",
@@ -120,7 +118,6 @@ def __init__(
         timezone: Optional[str] = configuration.WORKFLOW_TIME_ZONE,
         user: Optional[str] = configuration.WORKFLOW_USER,
         project: Optional[str] = configuration.WORKFLOW_PROJECT,
-        tenant: Optional[str] = configuration.WORKFLOW_TENANT,
         worker_group: Optional[str] = configuration.WORKFLOW_WORKER_GROUP,
         warning_type: Optional[str] = configuration.WORKFLOW_WARNING_TYPE,
         warning_group_id: Optional[int] = 0,
@@ -140,7 +137,6 @@ def __init__(
         self.timezone = timezone
         self._user = user
         self._project = project
-        self._tenant = tenant
         self.worker_group = worker_group
         self.warning_type = warning_type
         if warning_type.strip().upper() not in ("FAILURE", "SUCCESS", "ALL", "NONE"):
@@ -178,16 +174,6 @@ def __enter__(self) -> "Workflow":
     def __exit__(self, exc_type, exc_val, exc_tb) -> None:
         WorkflowContext.delete()
 
-    @property
-    def tenant(self) -> Tenant:
-        """Get attribute tenant."""
-        return Tenant(self._tenant)
-
-    @tenant.setter
-    def tenant(self, tenant: Tenant) -> None:
-        """Set attribute tenant."""
-        self._tenant = tenant.name
-
     @property
     def project(self) -> Project:
         """Get attribute project."""
@@ -204,7 +190,7 @@ def user(self) -> User:
 
         For now we just get from python models but not from java gateway models, so it may not correct.
         """
-        return User(name=self._user, tenant=self._tenant)
+        return User(name=self._user)
 
     @staticmethod
     def _parse_datetime(val: Any) -> Any:
@@ -438,7 +424,6 @@ def submit(self) -> int:
             self.execution_type,
             self.timeout,
             self.worker_group,
-            self._tenant,
             self.release_state,
             # TODO add serialization function
             json.dumps(self.task_relation_json),

diff --git a/src/pydolphinscheduler/default_config.yaml b/src/pydolphinscheduler/default_config.yaml
@@ -48,7 +48,6 @@ default:
   # Default value for dolphinscheduler's workflow object
   workflow:
     project: project-pydolphin
-    tenant: tenant_pydolphin
     user: userPythonGateway
     queue: queuePythonGateway
     worker_group: default

diff --git a/src/pydolphinscheduler/examples/bulk_create_example.py b/src/pydolphinscheduler/examples/bulk_create_example.py
@@ -31,8 +31,6 @@
 
 NUM_WORKFLOWS = 10
 NUM_TASKS = 5
-# Make sure your tenant exists in your operator system
-TENANT = "exists_tenant"
 # Whether task should dependent on pre one or not
 # False will create workflow with independent task, while True task will dependent on pre-task and dependence
 # link like `pre_task -> current_task -> next_task`, default True
@@ -41,7 +39,7 @@
 for wf in range(0, NUM_WORKFLOWS):
     workflow_name = f"workflow:{wf}"
 
-    with Workflow(name=workflow_name, tenant=TENANT) as workflow:
+    with Workflow(name=workflow_name) as workflow:
         for t in range(0, NUM_TASKS):
             task_name = f"task:{t}-{workflow_name}"
             command = f"echo This is task {task_name}"

diff --git a/src/pydolphinscheduler/examples/multi_resources_example.py b/src/pydolphinscheduler/examples/multi_resources_example.py
@@ -62,7 +62,6 @@
 
 with Workflow(
     name="multi_resources_example",
-    tenant="tenant_exists",
     # [start create_new_resources]
     resource_list=[
         Resource(

diff --git a/src/pydolphinscheduler/examples/task_condition_example.py b/src/pydolphinscheduler/examples/task_condition_example.py
@@ -35,7 +35,7 @@
 from pydolphinscheduler.tasks.condition import FAILURE, SUCCESS, And, Condition
 from pydolphinscheduler.tasks.shell import Shell
 
-with Workflow(name="task_condition_example", tenant="tenant_exists") as workflow:
+with Workflow(name="task_condition_example") as workflow:
     pre_task_1 = Shell(name="pre_task_1", command="echo pre_task_1")
     pre_task_2 = Shell(name="pre_task_2", command="echo pre_task_2")
     pre_task_3 = Shell(name="pre_task_3", command="echo pre_task_3")

diff --git a/src/pydolphinscheduler/examples/task_datax_example.py b/src/pydolphinscheduler/examples/task_datax_example.py
@@ -74,7 +74,6 @@
 
 with Workflow(
     name="task_datax_example",
-    tenant="tenant_exists",
 ) as workflow:
     # This task synchronizes the data in `t_ds_project`
     # of `first_mysql` database to `target_project` of `second_mysql` database.

diff --git a/src/pydolphinscheduler/examples/task_dependent_example.py b/src/pydolphinscheduler/examples/task_dependent_example.py
@@ -42,7 +42,6 @@
 
 with Workflow(
     name="task_dependent_external",
-    tenant="tenant_exists",
 ) as workflow:
     task_1 = Shell(name="task_1", command="echo task 1")
     task_2 = Shell(name="task_2", command="echo task 2")
@@ -51,7 +50,6 @@
 
 with Workflow(
     name="task_dependent_example",
-    tenant="tenant_exists",
 ) as workflow:
     task = Dependent(
         name="task_dependent",

diff --git a/src/pydolphinscheduler/examples/task_dvc_example.py b/src/pydolphinscheduler/examples/task_dvc_example.py
@@ -25,7 +25,6 @@
 
 with Workflow(
     name="task_dvc_example",
-    tenant="tenant_exists",
 ) as workflow:
     init_task = DVCInit(name="init_dvc", repository=repository, store_url="~/dvc_data")
     upload_task = DVCUpload(

diff --git a/src/pydolphinscheduler/examples/task_flink_example.py b/src/pydolphinscheduler/examples/task_flink_example.py
@@ -21,7 +21,7 @@
 from pydolphinscheduler.core.workflow import Workflow
 from pydolphinscheduler.tasks.flink import DeployMode, Flink, ProgramType
 
-with Workflow(name="task_flink_example", tenant="tenant_exists") as workflow:
+with Workflow(name="task_flink_example") as workflow:
     task = Flink(
         name="task_flink",
         main_class="org.apache.flink.streaming.examples.wordcount.WordCount",

diff --git a/src/pydolphinscheduler/examples/task_kubernetes_example.py b/src/pydolphinscheduler/examples/task_kubernetes_example.py
@@ -23,7 +23,6 @@
 
 with Workflow(
     name="task_kubernetes_example",
-    tenant="tenant_exists",
 ) as workflow:
     task_k8s = Kubernetes(
         name="task_k8s",

diff --git a/src/pydolphinscheduler/examples/task_map_reduce_example.py b/src/pydolphinscheduler/examples/task_map_reduce_example.py
@@ -22,7 +22,7 @@
 from pydolphinscheduler.core.workflow import Workflow
 from pydolphinscheduler.tasks.map_reduce import MR
 
-with Workflow(name="task_map_reduce_example", tenant="tenant_exists") as workflow:
+with Workflow(name="task_map_reduce_example") as workflow:
     task = MR(
         name="task_mr",
         main_class="wordcount",

diff --git a/src/pydolphinscheduler/examples/task_mlflow_example.py b/src/pydolphinscheduler/examples/task_mlflow_example.py
@@ -31,7 +31,6 @@
 
 with Workflow(
     name="task_mlflow_example",
-    tenant="tenant_exists",
 ) as workflow:
 
     # run custom mlflow project to train model

diff --git a/src/pydolphinscheduler/examples/task_openmldb_example.py b/src/pydolphinscheduler/examples/task_openmldb_example.py
@@ -29,7 +29,6 @@
 
 with Workflow(
     name="task_openmldb_example",
-    tenant="tenant_exists",
 ) as workflow:
     task_openmldb = OpenMLDB(
         name="task_openmldb",

diff --git a/src/pydolphinscheduler/examples/task_pytorch_example.py b/src/pydolphinscheduler/examples/task_pytorch_example.py
@@ -23,7 +23,6 @@
 
 with Workflow(
     name="task_pytorch_example",
-    tenant="tenant_exists",
 ) as workflow:
 
     # run project with existing environment

diff --git a/src/pydolphinscheduler/examples/task_sagemaker_example.py b/src/pydolphinscheduler/examples/task_sagemaker_example.py
@@ -35,7 +35,6 @@
 
 with Workflow(
     name="task_sagemaker_example",
-    tenant="tenant_exists",
 ) as workflow:
     task_sagemaker = SageMaker(
         name="task_sagemaker",

diff --git a/src/pydolphinscheduler/examples/task_spark_example.py b/src/pydolphinscheduler/examples/task_spark_example.py
@@ -21,7 +21,7 @@
 from pydolphinscheduler.core.workflow import Workflow
 from pydolphinscheduler.tasks.spark import DeployMode, ProgramType, Spark
 
-with Workflow(name="task_spark_example", tenant="tenant_exists") as workflow:
+with Workflow(name="task_spark_example") as workflow:
     task = Spark(
         name="task_spark",
         main_class="org.apache.spark.examples.SparkPi",

diff --git a/src/pydolphinscheduler/examples/task_switch_example.py b/src/pydolphinscheduler/examples/task_switch_example.py
@@ -34,9 +34,7 @@
 from pydolphinscheduler.tasks.shell import Shell
 from pydolphinscheduler.tasks.switch import Branch, Default, Switch, SwitchCondition
 
-with Workflow(
-    name="task_switch_example", tenant="tenant_exists", param={"var": "1"}
-) as workflow:
+with Workflow(name="task_switch_example", param={"var": "1"}) as workflow:
     parent = Shell(name="parent", command="echo parent")
     switch_child_1 = Shell(name="switch_child_1", command="echo switch_child_1")
     switch_child_2 = Shell(name="switch_child_2", command="echo switch_child_2")

diff --git a/src/pydolphinscheduler/examples/tutorial.py b/src/pydolphinscheduler/examples/tutorial.py
@@ -45,7 +45,6 @@
     name="tutorial",
     schedule="0 0 0 * * ? *",
     start_time="2021-01-01",
-    tenant="tenant_exists",
 ) as workflow:
     # [end workflow_declare]
     # [start task_declare]

diff --git a/src/pydolphinscheduler/examples/tutorial_decorator.py b/src/pydolphinscheduler/examples/tutorial_decorator.py
@@ -74,7 +74,6 @@ def task_union():
     name="tutorial_decorator",
     schedule="0 0 0 * * ? *",
     start_time="2021-01-01",
-    tenant="tenant_exists",
 ) as workflow:
     # [end workflow_declare]
 

diff --git a/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/src/pydolphinscheduler/examples/tutorial_resource_plugin.py
@@ -41,7 +41,6 @@
     name="tutorial_resource_plugin",
     schedule="0 0 0 * * ? *",
     start_time="2021-01-01",
-    tenant="tenant_exists",
     resource_plugin=Local("/tmp"),
 ) as workflow:
     # [end workflow_declare]

diff --git a/src/pydolphinscheduler/java_gateway.py b/src/pydolphinscheduler/java_gateway.py
@@ -258,7 +258,6 @@ def create_or_update_workflow(
         execution_type: str,
         timeout: int,
         worker_group: str,
-        tenant_code: str,
         release_state: int,
         task_relation_json: str,
         task_definition_json: str,
@@ -277,7 +276,6 @@ def create_or_update_workflow(
             warning_group_id,
             timeout,
             worker_group,
-            tenant_code,
             release_state,
             task_relation_json,
             task_definition_json,

diff --git a/src/pydolphinscheduler/models/tenant.py b/src/pydolphinscheduler/models/tenant.py
@@ -29,7 +29,7 @@ class Tenant(BaseSide):
 
     def __init__(
         self,
-        name: str = configuration.WORKFLOW_TENANT,
+        name: str = configuration.USER_TENANT,
         queue: str = configuration.WORKFLOW_QUEUE,
         description: Optional[str] = None,
         tenant_id: Optional[int] = None,