Skip to content

Commit

Permalink
update sdkv2 e2e tests for vision to use staging components (Azure#31438
Browse files Browse the repository at this point in the history 
)

* update sdkv2 e2e tests for vision and nlp to use staging components

* code reformatting using black

* restore nlp tests

* add params to component sweep settings

* remove feature flag and pipeline override from nlp tests

* Revert "remove feature flag and pipeline override from nlp tests"

This reverts commit e6a7511.
  • Loading branch information
@bhimar
bhimar authored Aug 10, 2023
1 parent 2026a95 commit c816234
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 243 deletions.
112 changes: 54 additions & 58 deletions sdk/ml/azure-ai-ml/tests/automl_job/e2etests/test_automl_image_classification.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,6 @@ def test_image_classification_multiclass_run(
training_data = Input(type=AssetTypes.MLTABLE, path=train_path)
validation_data = Input(type=AssetTypes.MLTABLE, path=val_path)

properties = get_automl_job_properties()
if components:
properties["_automl_subgraph_orchestration"] = "true"
properties[
"_pipeline_id_override"
] = "azureml://registries/azmlft-dev-registry01/components/image_classification_pipeline"

# Make generic classification job
image_classification_job = automl.image_classification(
training_data=training_data,
Expand All @@ -100,75 +93,78 @@ def test_image_classification_multiclass_run(
primary_metric="accuracy",
compute="gpu-cluster",
experiment_name="image-e2e-tests",
properties=properties,
properties=get_automl_job_properties(),
)

# Configure regular sweep job
# Configure sweep job
image_classification_job_sweep = copy.deepcopy(image_classification_job)
image_classification_job_sweep.set_training_parameters(early_stopping=True, evaluation_frequency=1)
image_classification_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["vitb16r224"]),
learning_rate=Uniform(0.001, 0.01),
number_of_epochs=Choice([1]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
layers_to_freeze=Choice([0, 2]),
number_of_epochs=Choice([1]),
),
]
)
image_classification_job_sweep.set_limits(max_trials=1, max_concurrent_trials=1)
image_classification_job_sweep.set_sweep(
sampling_algorithm="Random",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
image_classification_job_sweep.set_limits(max_trials=2, max_concurrent_trials=2)

# Configure AutoMode job
image_classification_job_automode = copy.deepcopy(image_classification_job)
# TODO: after shipping the AutoMode feature, do not set flag and call `set_limits()` instead of changing
# the limits object directly.
image_classification_job_automode.properties["enable_automode"] = True
image_classification_job_automode.limits.max_trials = 2
image_classification_job_automode.limits.max_concurrent_trials = 2

# Configure Finetune Sweep Job
if components:
image_classification_job_finetune_sweep = copy.deepcopy(image_classification_job)
image_classification_job_finetune_sweep.set_training_parameters(early_stopping=True, evaluation_frequency=1)
image_classification_job_finetune_sweep.extend_search_space(
# Configure components sweep job search space
image_classification_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["microsoft/beit-base-patch16-224"]),
learning_rate=Uniform(0.001, 0.01),
number_of_epochs=Choice([1]),
gradient_accumulation_step=Choice([1]),
learning_rate=Choice([0.005]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
# model-specific, valid_resize_size should be larger or equal than valid_crop_size
validation_resize_size=Choice([288]),
validation_crop_size=Choice([224]), # model-specific
training_crop_size=Choice([224]), # model-specific
number_of_epochs=Choice([1]),
),
]
)
image_classification_job_finetune_sweep.set_limits(max_trials=1, max_concurrent_trials=1)
image_classification_job_finetune_sweep.set_sweep(
image_classification_job_sweep.set_sweep(
sampling_algorithm="Grid",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
else:
# Configure runtime sweep job search space
image_classification_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["vitb16r224"]),
learning_rate=Uniform(0.005, 0.05),
number_of_epochs=Choice([1]),
gradient_accumulation_step=Choice([1, 2]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
learning_rate=Uniform(0.005, 0.05),
# model-specific, valid_resize_size should be larger or equal than valid_crop_size
validation_resize_size=Choice([288, 320, 352]),
validation_crop_size=Choice([224, 256]), # model-specific
training_crop_size=Choice([224, 256]), # model-specific
number_of_epochs=Choice([1]),
),
]
)
image_classification_job_sweep.set_sweep(
sampling_algorithm="Random",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
# Trigger finetune sweep
submitted_finetune_sweep = client.jobs.create_or_update(image_classification_job_finetune_sweep)

# Trigger regular sweep and then AutoMode job
submitted_job_sweep = client.jobs.create_or_update(image_classification_job_sweep)
submitted_job_automode = client.jobs.create_or_update(image_classification_job_automode)
# Configure AutoMode job
image_classification_job_automode = copy.deepcopy(image_classification_job)
image_classification_job_automode.set_limits(max_trials=2, max_concurrent_trials=2)

# Assert completion of finetune sweep job
if components:
assert_final_job_status(
submitted_finetune_sweep, client, ImageClassificationJob, JobStatus.COMPLETED, deadline=3600
)
# Trigger sweep job and then AutoMode job
submitted_job_sweep = client.jobs.create_or_update(image_classification_job_sweep)
if not components:
submitted_job_automode = client.jobs.create_or_update(image_classification_job_automode)

# Assert completion of regular sweep job
# Assert completion of sweep job
assert_final_job_status(submitted_job_sweep, client, ImageClassificationJob, JobStatus.COMPLETED, deadline=3600)

# Assert completion of Automode job
assert_final_job_status(
submitted_job_automode, client, ImageClassificationJob, JobStatus.COMPLETED, deadline=3600
)
if not components:
# Assert completion of Automode job
assert_final_job_status(
submitted_job_automode, client, ImageClassificationJob, JobStatus.COMPLETED, deadline=3600
)
116 changes: 52 additions & 64 deletions sdk/ml/azure-ai-ml/tests/automl_job/e2etests/test_automl_image_classification_multilabel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,6 @@ def test_image_classification_multilabel_run(
training_data = Input(type=AssetTypes.MLTABLE, path=train_path)
validation_data = Input(type=AssetTypes.MLTABLE, path=val_path)

properties = get_automl_job_properties()
if components:
properties["_automl_subgraph_orchestration"] = "true"
properties[
"_pipeline_id_override"
] = "azureml://registries/azmlft-dev-registry01/components/image_classification_pipeline"

# Make generic multilabel classification job
image_classification_multilabel_job = automl.image_classification_multilabel(
compute="gpu-cluster",
Expand All @@ -109,85 +102,80 @@ def test_image_classification_multilabel_run(
target_column_name="label",
primary_metric="iou",
# These are temporal properties needed in Private Preview
properties=properties,
properties=get_automl_job_properties(),
)

# Configure regular sweep job
# Configure sweep job
image_classification_multilabel_job_sweep = copy.deepcopy(image_classification_multilabel_job)
image_classification_multilabel_job_sweep.set_training_parameters(early_stopping=True, evaluation_frequency=1)
image_classification_multilabel_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["vitb16r224"]),
learning_rate=Uniform(0.005, 0.05),
number_of_epochs=Choice([1]),
gradient_accumulation_step=Choice([1, 2]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
learning_rate=Uniform(0.005, 0.05),
# model-specific, valid_resize_size should be larger or equal than valid_crop_size
validation_resize_size=Choice([288, 320, 352]),
validation_crop_size=Choice([224, 256]), # model-specific
training_crop_size=Choice([224, 256]), # model-specific
number_of_epochs=Choice([1]),
),
]
)
image_classification_multilabel_job_sweep.set_limits(max_trials=1, max_concurrent_trials=1)
image_classification_multilabel_job_sweep.set_sweep(
sampling_algorithm="Random",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
image_classification_multilabel_job_sweep.set_limits(max_trials=2, max_concurrent_trials=2)

# Configure AutoMode job
image_classification_multilabel_job_automode = copy.deepcopy(image_classification_multilabel_job)
# TODO: after shipping the AutoMode feature, do not set flag and call `set_limits()` instead of changing
# the limits object directly.
image_classification_multilabel_job_automode.properties["enable_automode"] = True
image_classification_multilabel_job_automode.limits.max_trials = 2
image_classification_multilabel_job_automode.limits.max_concurrent_trials = 2

# Configure Finetune Sweep Job
if components:
image_classification_multilabel_job_finetune_sweep = copy.deepcopy(image_classification_multilabel_job)
image_classification_multilabel_job_finetune_sweep.set_training_parameters(
early_stopping=True, evaluation_frequency=1
)
image_classification_multilabel_job_finetune_sweep.extend_search_space(
# Configure components sweep job search space
image_classification_multilabel_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["microsoft/beit-base-patch16-224"]),
number_of_epochs=Choice([1]),
gradient_accumulation_step=Choice([1]),
learning_rate=Choice([0.005]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
# model-specific, valid_resize_size should be larger or equal than valid_crop_size
validation_resize_size=Choice([288]),
validation_crop_size=Choice([224]), # model-specific
training_crop_size=Choice([224]), # model-specific
number_of_epochs=Choice([1]),
),
]
)
image_classification_multilabel_job_sweep.set_sweep(
sampling_algorithm="Grid",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
else:
# Configure runtime sweep job search space
image_classification_multilabel_job_sweep.extend_search_space(
[
SearchSpace(
model_name=Choice(["vitb16r224"]),
learning_rate=Uniform(0.005, 0.05),
number_of_epochs=Choice([1]),
gradient_accumulation_step=Choice([1, 2]),
),
SearchSpace(
model_name=Choice(["seresnext"]),
learning_rate=Uniform(0.005, 0.05),
# model-specific, valid_resize_size should be larger or equal than valid_crop_size
validation_resize_size=Choice([288, 320, 352]),
validation_crop_size=Choice([224, 256]), # model-specific
training_crop_size=Choice([224, 256]), # model-specific
number_of_epochs=Choice([1]),
),
]
)
image_classification_multilabel_job_finetune_sweep.set_limits(max_trials=1, max_concurrent_trials=1)
image_classification_multilabel_job_finetune_sweep.set_sweep(
image_classification_multilabel_job_sweep.set_sweep(
sampling_algorithm="Random",
early_termination=BanditPolicy(evaluation_interval=2, slack_factor=0.2, delay_evaluation=6),
)
# Trigger finetune sweep
submitted_finetune_sweep = client.jobs.create_or_update(image_classification_multilabel_job_finetune_sweep)

# Trigger regular sweep and then AutoMode job
submitted_job_sweep = client.jobs.create_or_update(image_classification_multilabel_job_sweep)
submitted_job_automode = client.jobs.create_or_update(image_classification_multilabel_job_automode)
# Configure AutoMode job
image_classification_multilabel_job_automode = copy.deepcopy(image_classification_multilabel_job)
image_classification_multilabel_job_automode.set_limits(max_trials=2, max_concurrent_trials=2)

# Assert completion of finetune sweep job
if components:
assert_final_job_status(
submitted_finetune_sweep, client, ImageClassificationMultilabelJob, JobStatus.COMPLETED, deadline=3600
)
# Trigger sweep job and then AutoMode job
submitted_job_sweep = client.jobs.create_or_update(image_classification_multilabel_job_sweep)
if not components:
submitted_job_automode = client.jobs.create_or_update(image_classification_multilabel_job_automode)

# Assert completion of regular sweep job
# Assert completion of sweep job
assert_final_job_status(
submitted_job_sweep, client, ImageClassificationMultilabelJob, JobStatus.COMPLETED, deadline=3600
)

# Assert completion of Automode job
assert_final_job_status(
submitted_job_automode, client, ImageClassificationMultilabelJob, JobStatus.COMPLETED, deadline=3600
)
if not components:
# Assert completion of Automode job
assert_final_job_status(
submitted_job_automode, client, ImageClassificationMultilabelJob, JobStatus.COMPLETED, deadline=3600
)
Loading

0 comments on commit c816234

Please sign in to comment.