addressing comments after sprint review

README.md

@@ -91,7 +91,7 @@ se_user_conf = {
     #Below two params are optional and need to be enabled to pass the custom email body
     #user_config.se_notifications_enable_custom_email_body: True,
     #user_config.se_notifications_email_custom_body: "Custom statistics: 'product_id': {}",
-    #Below parameter is optional and need to be enabled in case authorization is required to access smtp server.
+    #Below parameter is optional and needs to be enabled in case authorization is required to access smtp server.
     #user_config.se_notifications_email_smtp_auth: True,
 }
 ```

docs/bigquery.md

@@ -52,7 +52,7 @@ se: SparkExpectations = SparkExpectations(
     stats_table_writer=writer,
     target_and_error_table_writer=writer,
     debugger=False,
-    stats_streaming_options={user_config.se_enable_streaming: False},
+    stats_streaming_options={user_config.se_enable_streaming: False}
 )
 
 #if smtp server needs to be authenticated, password can be passed directly with user config or set in a secure way like cerberus or databricks secret

docs/delta.md

@@ -34,7 +34,6 @@ from spark_expectations.core.expectations import (
 )
 from spark_expectations.config.user_config import Constants as user_config
 
-
 writer = WrappedDataFrameWriter().mode("append").format("delta")
 
 se: SparkExpectations = SparkExpectations(
@@ -44,7 +43,7 @@ se: SparkExpectations = SparkExpectations(
     stats_table_writer=writer,
     target_and_error_table_writer=writer,
     debugger=False,
-    stats_streaming_options={user_config.se_enable_streaming: False},
+    stats_streaming_options={user_config.se_enable_streaming: False}
 )
 
 #if smtp server needs to be authenticated, password can be passed directly with user config or set in a secure way like cerberus or databricks secret

spark_expectations/config/user_config.py

@@ -21,6 +21,8 @@ class Constants:
     se_notifications_smtp_creds_dict = (
         "spark.expectations.notifications.smtp.creds.dict"
     )
+    cbs_smtp_password = "spark.expectations.notifications.cerberus.smtp.password"
+    dbx_smtp_password = "spark.expectations.notifications.dbx.smtp.password"
     se_notifications_email_from = "spark.expectations.notifications.email.from"
     se_notifications_email_to_other_mail_id = (
         "spark.expectations.notifications.email.to.other.mail.com"
@@ -72,7 +74,6 @@ class Constants:
     cbs_secret_app_name = "se.streaming.cbs.secret.app.name"
     cbs_secret_token = "se.streaming.cerberus.secret.token"
     cbs_topic_name = "se.streaming.cerberus.token.name"
-    cbs_smtp_password = "se.streaming.cerberus.smtp.password"
 
     dbx_workspace_url = "se.streaming.dbx.workspace.url"
     dbx_secret_scope = "se.streaming.dbx.secret.scope"
@@ -81,7 +82,6 @@ class Constants:
     dbx_secret_app_name = "se.streaming.dbx.secret.app.name"
     dbx_secret_token = "se.streaming.dbx.secret.token"
     dbx_topic_name = "se.streaming.dbx.topic.name"
-    dbx_smtp_password = "se.streaming.dbx.smtp.password"
 
     # declare const user config variables for agg query dq detailed stats
     se_enable_agg_dq_detailed_result = "spark.expectations.agg.dq.detailed.stats"

spark_expectations/examples/sample_dq_delta.py

@@ -23,7 +23,7 @@
 
 
 se: SparkExpectations = SparkExpectations(
-    product_id="your_productreport",
+    product_id="your_product",
     rules_df=spark.table("dq_spark_dev.dq_rules"),
     stats_table="dq_spark_dev.dq_stats",
     stats_table_writer=writer,

spark_expectations/notifications/plugins/email.py

@@ -96,7 +96,7 @@ def send_notification(
         function to send email notification for requested mail id's
         Args:
             _context: object of SparkExpectationsContext
-            _config_args: dict(which consist of: receiver mail(str), subject: subject of
+            _config_args: dict(which consists of: receiver mail(str), subject: subject of
                           the mail(str) and body: body of the mail(str)
         Returns:
 

tests/core/test_context.py

@@ -56,7 +56,7 @@ def test_context_properties():
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "abc",
-        "se.streaming.cerberus.smtp.password": "def"
+        "spark.expectations.notifications.cerberus.smtp.password": "def"
     }
     context._enable_mail = True
     context._enable_smtp_server_auth = True
@@ -189,7 +189,7 @@ def test_context_properties():
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "abc",
-        "se.streaming.cerberus.smtp.password": "def"
+        "spark.expectations.notifications.cerberus.smtp.password": "def"
     }
     assert context._enable_mail is True
     assert context._enable_smtp_server_auth is True
@@ -552,20 +552,20 @@ def test_set_smtp_creds_dict():
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "abc",
-        "se.streaming.cerberus.smtp.password": "def"
+        "spark.expectations.notifications.cerberus.smtp.password": "def"
     }
     )
     assert context._smtp_creds_dict == {
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "abc",
-        "se.streaming.cerberus.smtp.password": "def"
+        "spark.expectations.notifications.cerberus.smtp.password": "def"
     }
     assert context.get_smtp_creds_dict == {
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "abc",
-        "se.streaming.cerberus.smtp.password": "def"
+        "spark.expectations.notifications.cerberus.smtp.password": "def"
     }
 
 def test_set_to_mail():

tests/notification/plugins/test_email.py

@@ -114,7 +114,7 @@ def test_get_cerberus_password(_mock_secret_handler, _mock_get_secret):
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "app/project/env",
-        "se.streaming.cerberus.smtp.password": "password_key",
+        "spark.expectations.notifications.cerberus.smtp.password": "password_key",
     }
     _mock_get_secret.return_value = {
         "password_key": "test_password"
@@ -133,7 +133,7 @@ def test_get_cerberus_password_none(_mock_secret_handler, _mock_get_secret):
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "app/project/env",
-        "se.streaming.cerberus.smtp.password": None,
+        "spark.expectations.notifications.cerberus.smtp.password": None,
     }
     _mock_get_secret.return_value = None
     _mock_secret_handler.get_secret = _mock_get_secret
@@ -149,7 +149,7 @@ def test_get_databricks_password(_mock_secret_handler, _mock_get_secret):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
     _mock_get_secret.return_value = "test_password"
     _mock_secret_handler.get_secret = _mock_get_secret
@@ -182,7 +182,7 @@ def test_retrieve_password_cerberus(_mock_secret_handler):
         "se.streaming.secret.type": "cerberus",
         "se.streaming.cerberus.url": "https://xyz.com",
         "se.streaming.cerberus.sdb.path": "app/project/env",
-        "se.streaming.cerberus.smtp.password": "password_key",
+        "spark.expectations.notifications.cerberus.smtp.password": "password_key",
     }
     with patch("spark_expectations.notifications.plugins.email.SparkExpectationsEmailPluginImpl._get_cerberus_password") as mock_get_cerberus_password:
         email_handler._retrieve_password(_mock_secret_handler, secret_type, smtp_secret_dict)
@@ -197,7 +197,7 @@ def test_retrieve_password_databricks(_mock_secret_handler):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
     with patch("spark_expectations.notifications.plugins.email.SparkExpectationsEmailPluginImpl._get_databricks_password") as mock_get_databricks_password:
         email_handler._retrieve_password(_mock_secret_handler, secret_type, smtp_secret_dict)
@@ -212,7 +212,7 @@ def test_get_databricks_password(_mock_secret_handler, _mock_get_secret):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
     _mock_get_secret.return_value = "test_password"
     _mock_secret_handler.get_secret = _mock_get_secret
@@ -246,7 +246,7 @@ def test_retrieve_password_none(_mock_secret_handler):
         "se.streaming.secret.type": "not_databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
 
     password = email_handler._retrieve_password(_mock_secret_handler, secret_type, smtp_secret_dict)
@@ -275,7 +275,7 @@ def test_get_smtp_password_with_retrieve_method(_mock_context):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
 
     with patch("spark_expectations.notifications.plugins.email.smtplib.SMTP") as mock_smtp,\
@@ -294,7 +294,7 @@ def test_get_smtp_password_missing_key(_mock_context):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": None,
+        "spark.expectations.notifications.dbx.smtp.password": None,
     }
 
     with patch("spark_expectations.notifications.plugins.email.smtplib.SMTP") as mock_smtp, \
@@ -313,7 +313,7 @@ def test_get_smtp_password_failed_retrieve(_mock_context):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
 
     with patch("spark_expectations.notifications.plugins.email.smtplib.SMTP") as mock_smtp, \
@@ -333,7 +333,7 @@ def test_get_smtp_password_none_exception(_mock_context):
         "se.streaming.secret.type": "databricks",
         "se.streaming.dbx.workspace.url": "https://xyz.databricks.com",
         "se.streaming.dbx.secret.scope": "my_secret_scope",
-        "se.streaming.dbx.smtp.password": "password_key",
+        "spark.expectations.notifications.dbx.smtp.password": "password_key",
     }
 
     with patch("spark_expectations.notifications.plugins.email.smtplib.SMTP") as mock_smtp, \

tests/utils/test_reader.py

@@ -154,7 +154,7 @@ def fixture_product_rules_pipe():
             "se.streaming.secret.type": "cerberus",
             "se.streaming.cerberus.url": "https://xyz.com",
             "se.streaming.cerberus.sdb.path": "abc",
-            "se.streaming.cerberus.smtp.password": "def"
+            "spark.expectations.notifications.cerberus.smtp.password": "def"
                 },
          "spark.expectations.notifications.email.from": "sender@mail.com",
          "spark.expectations.notifications.email.to.other.mail.com": "recipient@mail.com",