[HDInsight]-Add livyspark api spec for hdi. (Azure#9469)

* Add livyspark api spec for hdi. * Prettier check * Update code style. * fix comments * prettier check * Update the code in order to unify the naming rules.
00Kai0 · Oct 12, 2020 · eae2839 · eae2839
1 parent 7319b2b
commit eae2839
Show file tree

Hide file tree

Showing 30 changed files with 1,600 additions and 22 deletions.
diff --git a/custom-words.txt b/custom-words.txt
@@ -1781,3 +1781,5 @@ userprincipalname
 sessionstate
 sessionhosts
 hostpool
+sparkr
+ambari
diff --git a/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkBatch.json b/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkBatch.json
@@ -0,0 +1,14 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "X-Requested-By": "admin",
+    "batchId": 123
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "msg": "deleted"
+      }
+    }
+  }
+}
diff --git a/...icrosoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkSessionJob.json b/...icrosoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkSessionJob.json
@@ -0,0 +1,14 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "x-Requested-By": "admin",
+    "sessionId": 123
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "msg": "deleted"
+      }
+    }
+  }
+}
diff --git a/...rosoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkStatementJob.json b/...rosoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_DeleteSparkStatementJob.json
@@ -0,0 +1,14 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "sessionId": 123,
+    "statementId": 123
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "msg": "canceled"
+      }
+    }
+  }
+}
diff --git a/...sight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_Get.json b/...sight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_Get.json
@@ -1,7 +1,7 @@
 {
   "parameters": {
     "user.name": "admin",
-    "clusterDnsName": "cluster.azurehdinsight.net",
+    "endpoint": "cluster.azurehdinsight.net",
     "jobId": "job_1542085634998_0024",
     "fields": "*"
   },
@@ -62,9 +62,7 @@
         "callback": null,
         "completed": "done",
         "userargs": {
-          "statusdir": "2018-11-13T05-49-52-4418fce1-e92e-4732-bc12-f2a3b8ef4432",
           "file": null,
-          "enablejobreconnect": null,
           "define": [],
           "enablelog": "false",
           "files": null,

diff --git a/...ta-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetAppState.json b/...ta-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetAppState.json
@@ -1,7 +1,7 @@
 {
   "parameters": {
     "appId": "application_1462754318084_0071",
-    "clusterDnsName": "cluster.azurehdinsight.net"
+    "endpoint": "cluster.azurehdinsight.net"
   },
   "responses": {
     "200": {

diff --git a/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchJob.json b/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchJob.json
@@ -0,0 +1,31 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "batchId": 45
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 45,
+        "appId": null,
+        "appInfo": {
+          "driverLogUrl": null,
+          "sparkUiUrl": null
+        },
+        "state": "starting",
+        "log": [
+          "\t queue: default",
+          "\t start time: 1590386942572",
+          "\t final status: UNDEFINED",
+          "\t tracking URL: https://shangwei-hdi-cmk-ps.azurehdinsight.net/yarnui/hn/proxy/application_1590286636717_0005/",
+          "\t user: livy",
+          "20/05/25 06:09:02 INFO ShutdownHookManager: Shutdown hook called",
+          "20/05/25 06:09:02 INFO ShutdownHookManager: Deleting directory /tmp/spark-9ddb6d73-f204-44a4-83e8-afdbd4ce1a0b",
+          "20/05/25 06:09:02 INFO ShutdownHookManager: Deleting directory /tmp/spark-ef198a0d-de24-4da9-aeb5-c1b78c5fdd5c",
+          "\nstderr: ",
+          "\nYARN Diagnostics: "
+        ]
+      }
+    }
+  }
+}
diff --git a/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchLog.json b/...ane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchLog.json
@@ -0,0 +1,29 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "batchId": 45,
+    "from": 1,
+    "size": 10
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 45,
+        "from": 1,
+        "size": 10,
+        "log": [
+          "SLF4J: Class path contains multiple SLF4J bindings.",
+          "SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.3015-8/spark2/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]",
+          "SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.3015-8/spark_llap/spark-llap-assembly-1.0.0.2.6.5.3015-8.jar!/org/slf4j/impl/StaticLoggerBinder.class]",
+          "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.",
+          "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]",
+          "Warning: Master yarn-cluster is deprecated since 2.0. Please use master \"yarn\" with specified deploy mode instead.",
+          "20/05/25 06:08:59 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable",
+          "Warning: Skip remote jar wasbs://containercmk002@shangweistorageaccount.blob.core.windows.net/microsoft-spark-2.3.x-0.6.0.jar.",
+          "20/05/25 06:08:59 INFO MetricsConfig: loaded properties from hadoop-metrics2-azure-file-system.properties",
+          "20/05/25 06:08:59 INFO WasbAzureIaasSink: Init starting."
+        ]
+      }
+    }
+  }
+}
diff --git a/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchState.json b/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkBatchState.json
@@ -0,0 +1,14 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "batchId": 45
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 45,
+        "state": "starting"
+      }
+    }
+  }
+}
diff --git a/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionJob.json b/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionJob.json
@@ -0,0 +1,34 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "sessionId": 123
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 34,
+        "appId": "application_1590286636717_0004",
+        "owner": null,
+        "proxyUser": null,
+        "state": "idle",
+        "kind": "spark",
+        "appInfo": {
+          "driverLogUrl": "http://wn2-shangw.4jhwm2oxfmoehnhvt2gwwtriqb.bx.internal.cloudapp.net:30060/node/containerlogs/container_e06_1590286636717_0004_01_000001/livy",
+          "sparkUiUrl": "https://shangwei-hdi-cmk-ps.azurehdinsight.net/yarnui/hn/proxy/application_1590286636717_0004/"
+        },
+        "log": [
+          "\t ApplicationMaster RPC port: -1",
+          "\t queue: default",
+          "\t start time: 1590384019312",
+          "\t final status: UNDEFINED",
+          "\t tracking URL: https://shangwei-hdi-cmk-ps.azurehdinsight.net/yarnui/hn/proxy/application_1590286636717_0004/",
+          "\t user: livy",
+          "20/05/25 05:20:19 INFO ShutdownHookManager: Shutdown hook called",
+          "20/05/25 05:20:19 INFO ShutdownHookManager: Deleting directory /tmp/spark-1557a190-880c-422d-a744-ce31d0fefa1d",
+          "20/05/25 05:20:19 INFO ShutdownHookManager: Deleting directory /tmp/spark-12bb0346-a2c9-4b9d-9f9b-feb91e30f554",
+          "\nYARN Diagnostics: "
+        ]
+      }
+    }
+  }
+}
diff --git a/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionLog.json b/...e/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionLog.json
@@ -0,0 +1,29 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "sessionId": 34,
+    "from": 1,
+    "size": 10
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 34,
+        "from": 0,
+        "total": 57,
+        "log": [
+          "stdout: ",
+          "\nstderr: ",
+          "SLF4J: Class path contains multiple SLF4J bindings.",
+          "SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.3015-8/spark2/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]",
+          "SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.3015-8/spark_llap/spark-llap-assembly-1.0.0.2.6.5.3015-8.jar!/org/slf4j/impl/StaticLoggerBinder.class]",
+          "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.",
+          "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]",
+          "Warning: Master yarn-cluster is deprecated since 2.0. Please use master \"yarn\" with specified deploy mode instead.",
+          "20/05/25 05:20:14 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable",
+          "20/05/25 05:20:15 INFO MetricsConfig: loaded properties from hadoop-metrics2-azure-file-system.properties"
+        ]
+      }
+    }
+  }
+}
diff --git a/...Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionState.json b/...Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkSessionState.json
@@ -0,0 +1,14 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "sessionId": 34
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 34,
+        "state": "idle"
+      }
+    }
+  }
+}
diff --git a/...Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkStatementJob.json b/...Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_GetSparkStatementJob.json
@@ -0,0 +1,24 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "sessionId": 34,
+    "statementId": 1
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": 0,
+        "code": "1 + 1",
+        "state": "available",
+        "output": {
+          "status": "ok",
+          "execution_count": 0,
+          "data": {
+            "text/plain": "res0: Int = 2"
+          }
+        },
+        "progress": 1.0
+      }
+    }
+  }
+}
diff --git a/...ight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_Kill.json b/...ight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_Kill.json
@@ -1,7 +1,7 @@
 {
   "parameters": {
     "user.name": "admin",
-    "clusterDnsName": "cluster.azurehdinsight.net",
+    "endpoint": "cluster.azurehdinsight.net",
     "jobId": "job_1542085634998_0007"
   },
   "responses": {
@@ -62,7 +62,6 @@
         "completed": null,
         "userargs": {
           "statusdir": "2018-11-13T05-35-01-6f79af0f-6157-494a-a03e-eabb190a0c10",
-          "enablejobreconnect": null,
           "arg": [
             "10",
             "1000"
@@ -73,11 +72,9 @@
             "mapreduce.task.timeout=60000"
           ],
           "enablelog": "false",
-          "libjars": null,
           "files": null,
           "callback": null,
-          "jar": "/example/jars/hadoop-mapreduce-examples.jar",
-          "class": "pi"
+          "jar": "/example/jars/hadoop-mapreduce-examples.jar"
         },
         "msg": null
       }

diff --git a/...ight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_List.json b/...ight/data-plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_List.json
@@ -1,7 +1,7 @@
 {
   "parameters": {
     "user.name": "admin",
-    "clusterDnsName": "cluster.azurehdinsight.net",
+    "endpoint": "cluster.azurehdinsight.net",
     "fields": "*",
     "showall": "true"
   },

diff --git a/...plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListAfterJobId.json b/...plane/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListAfterJobId.json
@@ -1,7 +1,7 @@
 {
   "parameters": {
     "user.name": "admin",
-    "clusterDnsName": "cluster.azurehdinsight.net",
+    "endpoint": "cluster.azurehdinsight.net",
     "jobId": "job_1542085634998_0029",
     "numrecords": 3,
     "fields": "*",

diff --git a/...ne/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListSparkBatchJob.json b/...ne/Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListSparkBatchJob.json
@@ -0,0 +1,38 @@
+{
+  "parameters": {
+    "endpoint": "clustername.azurehdinsight.net",
+    "from": 0,
+    "size": 1
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "from": 0,
+        "total": 1,
+        "sessions": [
+          {
+            "id": 44,
+            "state": "starting",
+            "appId": null,
+            "appInfo": {
+              "driverLogUrl": null,
+              "sparkUiUrl": null
+            },
+            "log": [
+              "20/05/25 04:59:18 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable",
+              "Warning: Skip remote jar wasbs://containercmk002@shangweistorageaccount.blob.core.windows.net/microsoft-spark-2.3.x-0.6.0.jar.",
+              "20/05/25 04:59:18 INFO MetricsConfig: loaded properties from hadoop-metrics2-azure-file-system.properties",
+              "20/05/25 04:59:18 INFO WasbAzureIaasSink: Init starting.",
+              "20/05/25 04:59:18 INFO AzureIaasSink: Init starting. Initializing MdsLogger.",
+              "20/05/25 04:59:18 INFO AzureIaasSink: Init completed.",
+              "20/05/25 04:59:18 INFO WasbAzureIaasSink: Init completed.",
+              "20/05/25 04:59:18 INFO MetricsSinkAdapter: Sink azurefs2 started",
+              "\nstderr: ",
+              "\nYARN Diagnostics: "
+            ]
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/.../Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListSparkSessionJob.json b/.../Microsoft.HDInsight/preview/2018-11-01-preview/examples/HDI_Job_ListSparkSessionJob.json
@@ -0,0 +1,41 @@
+{
+  "parameters": {
+    "endpoint": "cluster.azurehdinsight.net",
+    "from": 0,
+    "size": 1
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "from": 0,
+        "total": 1,
+        "sessions": [
+          {
+            "id": 34,
+            "appId": "application_1590286636717_0004",
+            "owner": null,
+            "proxyUser": null,
+            "state": "idle",
+            "kind": "spark",
+            "appInfo": {
+              "driverLogUrl": "http://wn2-shangw.4jhwm2oxfmoehnhvt2gwwtriqb.bx.internal.cloudapp.net:30060/node/containerlogs/container_e06_1590286636717_0004_01_000001/livy",
+              "sparkUiUrl": "https://shangwei-hdi-cmk-ps.azurehdinsight.net/yarnui/hn/proxy/application_1590286636717_0004/"
+            },
+            "log": [
+              "\t ApplicationMaster RPC port: -1",
+              "\t queue: default",
+              "\t start time: 1590384019312",
+              "\t final status: UNDEFINED",
+              "\t tracking URL: https://shangwei-hdi-cmk-ps.azurehdinsight.net/yarnui/hn/proxy/application_1590286636717_0004/",
+              "\t user: livy",
+              "20/05/25 05:20:19 INFO ShutdownHookManager: Shutdown hook called",
+              "20/05/25 05:20:19 INFO ShutdownHookManager: Deleting directory /tmp/spark-1557a190-880c-422d-a744-ce31d0fefa1d",
+              "20/05/25 05:20:19 INFO ShutdownHookManager: Deleting directory /tmp/spark-12bb0346-a2c9-4b9d-9f9b-feb91e30f554",
+              "\nYARN Diagnostics: "
+            ]
+          }
+        ]
+      }
+    }
+  }
+}