From 85230a2d954429afc5bc4b637e30e647f3815d19 Mon Sep 17 00:00:00 2001 From: zhangxingzhi Date: Mon, 17 Oct 2022 21:59:42 +0800 Subject: [PATCH] fix: fix ci --- .../internal/unittests/test_pipeline_job.py | 17 ++- .../unittests/test_pipeline_job_entity.py | 1 - .../pipeline_job_with_properties.yml | 140 +----------------- .../pipeline_job_with_properties.yml | 12 -- 4 files changed, 23 insertions(+), 147 deletions(-) diff --git a/sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py b/sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py index c4d954ee3769..123b46f1c53c 100644 --- a/sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py +++ b/sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py @@ -8,7 +8,7 @@ import pytest import yaml -from azure.ai.ml import Input, load_component +from azure.ai.ml import Input, load_component, load_job from azure.ai.ml._internal import ( AISuperComputerConfiguration, AISuperComputerScalePolicy, @@ -592,3 +592,18 @@ def test_pipeline_with_setting_node_output_directly(self) -> None: copy_file.outputs.output_dir.path = "path_on_datastore" assert copy_file.outputs.output_dir.path == "path_on_datastore" assert copy_file.outputs.output_dir.type == "path" + + def test_job_properties(self): + pipeline_job: PipelineJob = load_job( + source="./tests/test_configs/internal/pipeline_jobs/pipeline_job_with_properties.yml" + ) + pipeline_dict = pipeline_job._to_dict() + rest_pipeline_dict = pipeline_job._to_rest_object().as_dict()["properties"] + assert pipeline_dict["properties"] == {"AZURE_ML_PathOnCompute_input_data": "/tmp/test"} + assert rest_pipeline_dict["properties"] == pipeline_dict["properties"] + for name, node_dict in pipeline_dict["jobs"].items(): + rest_node_dict = rest_pipeline_dict["jobs"][name] + assert len(node_dict["properties"]) == 1 + assert "AZURE_ML_PathOnCompute_" in list(node_dict["properties"].keys())[0] + assert node_dict["properties"] == rest_node_dict["properties"] + diff --git a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py index 0108e38b4562..e06d755f47b4 100644 --- a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py +++ b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py @@ -1434,7 +1434,6 @@ def test_non_string_pipeline_node_input(self): "type": "command", } - @pytest.mark.usefixtures("enable_internal_components") def test_job_properties(self): pipeline_job: PipelineJob = load_job( source="./tests/test_configs/pipeline_jobs/pipeline_job_with_properties.yml" diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/internal/pipeline_jobs/pipeline_job_with_properties.yml b/sdk/ml/azure-ai-ml/tests/test_configs/internal/pipeline_jobs/pipeline_job_with_properties.yml index a208c528b32f..3904529ce48f 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/internal/pipeline_jobs/pipeline_job_with_properties.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/internal/pipeline_jobs/pipeline_job_with_properties.yml @@ -26,139 +26,13 @@ properties: AZURE_ML_PathOnCompute_input_data: "/tmp/test" jobs: - node0: # inline command job with properties - command: echo hello ${{inputs.hello_string}} - environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu@latest - inputs: - hello_string: ${{parent.inputs.hello_string}} - properties: - AZURE_ML_PathOnCompute_hello_string: "/tmp/test" - - node1: # inline parallel job with properties - type: parallel - compute: "azureml:cpu-cluster" - inputs: - test1: ${{parent.inputs.input_data}} - resources: - instance_count: 3 - mini_batch_size: "100kb" - mini_batch_error_threshold: 5 - logging_level: "DEBUG" - input_data: ${{inputs.input_data}} - max_concurrency_per_instance: 2 - task: - type: run_function - code: "../python" - entry_script: pass_through.py - append_row_to: ${{outputs.scored_result}} # optional, If Null, equals to summary_only mode in v1. - environment: azureml:my-env:1 - properties: - AZURE_ML_PathOnCompute_input_data: "/tmp/test" - - node2: # inline import job with properties - type: import - source: - type: azuresqldb - query: >- - select * from REGION - connection: azureml:my_username_password - output: - type: mltable - path: azureml://datastores/workspaceblobstore/paths/output_dir/ - properties: - AZURE_ML_PathOnCompute_output: "/tmp/test" - - node3: # inline spark job with properties - type: spark - inputs: - test1: ${{parent.inputs.input_data}} - file_input2: ${{parent.inputs.input_data}} - code: ../dsl_pipeline/spark_job_in_pipeline/src - entry: - file: entry.py # file path of the entry file relative to the code root folder - py_files: - - utils.zip - jars: - - scalaproj.jar - files: - - my_files.txt - args: >- - --file_input1 ${{inputs.test1}} - --file_input2 ${{inputs.file_input2}} - --output ${{outputs.output}} - compute: azureml:rezas-synapse-10 - conf: - spark.driver.cores: 2 - spark.driver.memory: "1g" - spark.executor.cores: 1 - spark.executor.memory: "1g" - spark.executor.instances: 1 - properties: - AZURE_ML_PathOnCompute_input_data: "/tmp/test" - - node4: # inline automl job with properties - type: automl - task: text_ner - log_verbosity: info - primary_metric: accuracy - limits: - max_trials: 1 - timeout_minutes: 60 - training_data: ${{parent.inputs.text_ner_training_data}} - validation_data: ${{parent.inputs.text_ner_validation_data}} - properties: - AZURE_ML_PathOnCompute_training_data: "/tmp/test" - - node5: # inline sweep job with properties - type: sweep - search_space: - component_in_number: - type: choice - values: - - 25 - - 35 - limits: - max_total_trials: 3 - sampling_algorithm: random - objective: - goal: maximize - primary_metric: accuracy - trial: azureml:microsoftsamplescommandcomponentbasic_nopaths_test:1 - properties: - AZURE_ML_PathOnCompute_input: "/tmp/test" - - node6: # parallel node with properties as a typical implement of base node. - type: parallel + node7: # internal command node with properties as a typical implement of internal base node. + type: CommandComponent compute: azureml:cpu-cluster - component: ../components/parallel_component_with_file_input.yml + component: file:../helloworld/helloworld_component_command.yml inputs: - job_data_path: ${{parent.inputs.pipeline_job_data_path}} - outputs: - job_output_path: - mini_batch_size: "1" - mini_batch_error_threshold: 1 - max_concurrency_per_instance: 1 - properties: - AZURE_ML_PathOnCompute_job_data_path: "/tmp/test" - -# Comment these lines out as internal node is not well supported in yaml now. -# node7: # internal command node with properties as a typical implement of internal base node. -# type: CommandComponent -# compute: azureml:cpu-cluster -# component: ../internal/helloworld/helloworld_component_command.yml -# inputs: -# training_data: ${{parent.inputs.input_data}} -# max_epochs: 10 -# learning_rate: 0.01 -# properties: -# AZURE_ML_PathOnCompute_job_training_data: "/tmp/test" - - node8: # pipeline node with properties - type: pipeline - inputs: - component_in_number: 11 - component_in_path: ${{parent.inputs.input_data}} - - component: ../components/helloworld_pipeline_component.yml + training_data: ${{parent.inputs.input_data}} + max_epochs: 10 + learning_rate: 0.01 properties: - AZURE_ML_PathOnCompute_job_component_in_path: "/tmp/test" + AZURE_ML_PathOnCompute_job_training_data: "/tmp/test" diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/pipeline_job_with_properties.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/pipeline_job_with_properties.yml index fd9201d83206..6f686cef5891 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/pipeline_job_with_properties.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/pipeline_job_with_properties.yml @@ -141,18 +141,6 @@ jobs: properties: AZURE_ML_PathOnCompute_job_data_path: "/tmp/test" -# Comment these lines out as internal node is not well supported in yaml now. - node7: # internal command node with properties as a typical implement of internal base node. - type: CommandComponent - compute: azureml:cpu-cluster - component: file:../internal/helloworld/helloworld_component_command.yml - inputs: - training_data: ${{parent.inputs.input_data}} - max_epochs: 10 - learning_rate: 0.01 - properties: - AZURE_ML_PathOnCompute_job_training_data: "/tmp/test" - node8: # pipeline node with properties type: pipeline inputs: