-
Notifications
You must be signed in to change notification settings - Fork 0
/
My_Notebook2.html
39 lines (37 loc) · 125 KB
/
My_Notebook2.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
<!DOCTYPE html>
<html>
<head>
<meta name="databricks-html-version" content="1">
<title>My_Notebook - Databricks</title>
<meta charset="utf-8">
<meta name="google" content="notranslate">
<meta name="robots" content="nofollow">
<meta http-equiv="Content-Language" content="en">
<meta http-equiv="Content-Type" content="text/html; charset=UTF8">
<link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/css/main.css">
<link rel="stylesheet" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/css/print.css" media="print">
<link rel="icon" type="image/png" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/img/favicon.ico"/>
<script>window.settings = {"enableUsageDeliveryConfiguration":false,"enableNotebookNotifications":true,"enableSshKeyUI":false,"defaultInteractivePricePerDBU":0.4,"enableClusterMetricsUI":true,"enableOnDemandClusterType":true,"enableAutoCompleteAsYouType":[],"devTierName":"Community Edition","enableJobsPrefetching":true,"workspaceFeaturedLinks":[{"linkURI":"https://docs.databricks.com/index.html","displayName":"Documentation","icon":"question"},{"linkURI":"https://docs.databricks.com/release-notes/product/index.html","displayName":"Release Notes","icon":"code"},{"linkURI":"https://docs.databricks.com/spark/latest/training/index.html","displayName":"Training & Tutorials","icon":"graduation-cap"}],"enableReservoirTableUI":false,"enableClearStateFeature":true,"dbcForumURL":"http://forums.databricks.com/","enableProtoClusterInfoDeltaPublisher":true,"enableAttachExistingCluster":true,"resetJobListOnConnect":true,"serverlessDefaultSparkVersion":"latest-stable-scala2.11","maxCustomTags":45,"serverlessDefaultMaxWorkers":20,"enableInstanceProfilesUIInJobs":true,"nodeInfo":{"node_types":[{"support_ssh":false,"spark_heap_memory":4800,"instance_type_id":"r3.2xlarge","spark_core_oversubscription_factor":8.0,"node_type_id":"dev-tier-node","description":"Community Optimized","support_cluster_tags":false,"container_memory_mb":6000,"node_instance_type":{"instance_type_id":"r3.2xlarge","provider":"AWS","local_disk_size_gb":160,"compute_units":26.0,"number_of_ips":14,"local_disks":1,"reserved_compute_units":3.64,"gpus":0,"memory_mb":62464,"num_cores":8,"local_disk_type":"AHCI","max_attachable_disks":0,"supported_disk_types":[{"ebs_volume_type":"GENERAL_PURPOSE_SSD"},{"ebs_volume_type":"THROUGHPUT_OPTIMIZED_HDD"}],"reserved_memory_mb":4800},"memory_mb":6144,"is_hidden":false,"category":"Community Edition","num_cores":0.88,"support_port_forwarding":false,"support_ebs_volumes":false,"is_deprecated":false}],"default_node_type_id":"dev-tier-node"},"sqlAclsDisabledMap":{"spark.databricks.acl.enabled":"false","spark.databricks.acl.sqlOnly":"false"},"enableDatabaseSupportClusterChoice":true,"enableClusterAcls":true,"notebookRevisionVisibilityHorizon":999999,"serverlessClusterProductName":"Serverless Pool","showS3TableImportOption":true,"maxEbsVolumesPerInstance":10,"enableRStudioUI":false,"isAdmin":true,"deltaProcessingBatchSize":1000,"timerUpdateQueueLength":100,"sqlAclsEnabledMap":{"spark.databricks.acl.enabled":"true","spark.databricks.acl.sqlOnly":"true"},"enableLargeResultDownload":true,"maxElasticDiskCapacityGB":5000,"serverlessDefaultMinWorkers":2,"zoneInfos":[{"id":"us-west-2c","isDefault":true},{"id":"us-west-2b","isDefault":false},{"id":"us-west-2a","isDefault":false}],"enableCustomSpotPricingUIByTier":false,"serverlessClustersEnabled":false,"enableWorkspaceBrowserSorting":true,"enableSentryLogging":false,"enableFindAndReplace":true,"disallowUrlImportExceptFromDocs":false,"defaultStandardClusterModel":{"cluster_name":"","node_type_id":"dev-tier-node","spark_version":"3.5.x-scala2.11","num_workers":0,"aws_attributes":{"first_on_demand":0,"availability":"ON_DEMAND","zone_id":"us-west-2c","spot_bid_price_percent":100},"autotermination_minutes":120,"default_tags":{"Vendor":"Databricks","Creator":"soumendas0123@gmail.com","ClusterName":null,"ClusterId":"<Generated after creation>"}},"enableEBSVolumesUIForJobs":true,"enablePublishNotebooks":true,"enableBitbucketCloud":true,"createTableInNotebookS3Link":{"url":"https://docs.databricks.com/_static/notebooks/data-import/s3.html","displayName":"S3","workspaceFileName":"S3 Example"},"sanitizeHtmlResult":true,"enableJobAclsConfig":false,"enableFullTextSearch":false,"enableElasticSparkUI":false,"enableNewClustersCreate":true,"clusters":true,"allowRunOnPendingClusters":true,"useAutoscalingByDefault":false,"enableAzureToolbar":false,"fileStoreBase":"FileStore","enableEmailInAzure":false,"enableRLibraries":true,"enableTableAclsConfig":false,"enableSshKeyUIInJobs":true,"enableDetachAndAttachSubMenu":true,"configurableSparkOptionsSpec":[{"keyPattern":"spark\\.kryo(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.kryo.*","valuePatternDisplay":"*","description":"Configuration options for Kryo serialization"},{"keyPattern":"spark\\.io\\.compression\\.codec","valuePattern":"(lzf|snappy|org\\.apache\\.spark\\.io\\.LZFCompressionCodec|org\\.apache\\.spark\\.io\\.SnappyCompressionCodec)","keyPatternDisplay":"spark.io.compression.codec","valuePatternDisplay":"snappy|lzf","description":"The codec used to compress internal data such as RDD partitions, broadcast variables and shuffle outputs."},{"keyPattern":"spark\\.serializer","valuePattern":"(org\\.apache\\.spark\\.serializer\\.JavaSerializer|org\\.apache\\.spark\\.serializer\\.KryoSerializer)","keyPatternDisplay":"spark.serializer","valuePatternDisplay":"org.apache.spark.serializer.JavaSerializer|org.apache.spark.serializer.KryoSerializer","description":"Class to use for serializing objects that will be sent over the network or need to be cached in serialized form."},{"keyPattern":"spark\\.rdd\\.compress","valuePattern":"(true|false)","keyPatternDisplay":"spark.rdd.compress","valuePatternDisplay":"true|false","description":"Whether to compress serialized RDD partitions (e.g. for StorageLevel.MEMORY_ONLY_SER). Can save substantial space at the cost of some extra CPU time."},{"keyPattern":"spark\\.speculation","valuePattern":"(true|false)","keyPatternDisplay":"spark.speculation","valuePatternDisplay":"true|false","description":"Whether to use speculation (recommended off for streaming)"},{"keyPattern":"spark\\.es(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"es(\\.([^\\.]+))+","valuePattern":".*","keyPatternDisplay":"es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"spark\\.(storage|shuffle)\\.memoryFraction","valuePattern":"0?\\.0*([1-9])([0-9])*","keyPatternDisplay":"spark.(storage|shuffle).memoryFraction","valuePatternDisplay":"(0.0,1.0)","description":"Fraction of Java heap to use for Spark's shuffle or storage"},{"keyPattern":"spark\\.streaming\\.backpressure\\.enabled","valuePattern":"(true|false)","keyPatternDisplay":"spark.streaming.backpressure.enabled","valuePatternDisplay":"true|false","description":"Enables or disables Spark Streaming's internal backpressure mechanism (since 1.5). This enables the Spark Streaming to control the receiving rate based on the current batch scheduling delays and processing times so that the system receives only as fast as the system can process. Internally, this dynamically sets the maximum receiving rate of receivers. This rate is upper bounded by the values `spark.streaming.receiver.maxRate` and `spark.streaming.kafka.maxRatePerPartition` if they are set."},{"keyPattern":"spark\\.streaming\\.receiver\\.maxRate","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.receiver.maxRate","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which each receiver will receive data. Effectively, each stream will consume at most this number of records per second. Setting this configuration to 0 or a negative number will put no limit on the rate. See the deployment guide in the Spark Streaming programing guide for mode details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRatePerPartition","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRatePerPartition","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which data will be read from each Kafka partition when using the Kafka direct stream API introduced in Spark 1.3. See the Kafka Integration guide for more details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRetries","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRetries","valuePatternDisplay":"numeric","description":"Maximum number of consecutive retries the driver will make in order to find the latest offsets on the leader of each partition (a default value of 1 means that the driver will make a maximum of 2 attempts). Only applies to the Kafka direct stream API introduced in Spark 1.3."},{"keyPattern":"spark\\.streaming\\.ui\\.retainedBatches","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.ui.retainedBatches","valuePatternDisplay":"numeric","description":"How many batches the Spark Streaming UI and status APIs remember before garbage collecting."}],"enableReactNotebookComments":true,"enableAdminPasswordReset":false,"checkBeforeAddingAadUser":false,"enableResetPassword":true,"maxClusterTagValueLength":255,"enableJobsSparkUpgrade":true,"createTableInNotebookDBFSLink":{"url":"https://docs.databricks.com/_static/notebooks/data-import/dbfs.html","displayName":"DBFS","workspaceFileName":"DBFS Example"},"perClusterAutoterminationEnabled":false,"enableNotebookCommandNumbers":true,"allowStyleInSanitizedHtml":true,"sparkVersions":[{"key":"1.6.3-db2-hadoop2-scala2.10","displayName":"Spark 1.6.3-db2 (Hadoop 2, Scala 2.10)","packageLabel":"spark-image-aba860a0ffce4f3471fb14aefdcb1d768ac66a53a5ad884c48745ef98aeb9d67","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"3.3.x-gpu-scala2.11","displayName":"3.3 (includes Apache Spark 2.2.0, GPU, Scala 2.11)","packageLabel":"spark-image-86b4917bb6586289ca64e65f64fd23678c297274be6cd6aa6aa01d7b91fed29c","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.1.1-db5-scala2.11","displayName":"Spark 2.1.1-db5 (Scala 2.11)","packageLabel":"spark-image-08d9fc1551087e0876236f19640c4a83116b1649f15137427d21c9056656e80e","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"1.6.x-ubuntu15.10","displayName":"Spark 1.6.x (Hadoop 1)","packageLabel":"spark-image-8cea23fb9094e174bf5815d79009f4a8e383eb86cf2909cf6c6434ed8da2a16a","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.3.x-scala2.10","displayName":"3.3 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-d7df74e188103a4093ff4467dbf0d32886366c984097f6997e0cd87d0f6b2fa5","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"1.4.x-ubuntu15.10","displayName":"Spark 1.4.1 (Hadoop 1, deprecated)","packageLabel":"spark-image-f710650fb8aaade8e4e812368ea87c45cd8cd0b5e6894ca6c94f3354e8daa6dc","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.2.x-scala2.11","displayName":"3.0 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-67ab3a06d1e83d5b60df7063245eb419a2e9fe329aeeb7e7d9713332c669bb17","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.1.1-db6-scala2.10","displayName":"Spark 2.1.1-db6 (Scala 2.10)","packageLabel":"spark-image-177f3f02a6a3432d30068332dc857b9161345bdd2ee8a2d2de05bb05cb4b0f4c","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.1.0-db2-scala2.11","displayName":"Spark 2.1.0-db2 (Scala 2.11)","packageLabel":"spark-image-267c4490a3ab8a39acdbbd9f1d36f6decdecebf013e30dd677faff50f1d9cf8b","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"4.0.x-scala2.11","displayName":"4.0 beta (Scala 2.11)","packageLabel":"spark-image-0dbee48aa83ff77df1e7b2892d90c671c7b31f94e9fd724e17b9dc4153e12df3","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"2.1.x-gpu-scala2.11","displayName":"Spark 2.1 (Auto-updating, GPU, Scala 2.11 experimental)","packageLabel":"spark-image-d613235f93e0f29838beb2079a958c02a192ed67a502192bc67a8a5f2fb37f35","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.0.0-ubuntu15.10-scala2.10","displayName":"Spark 2.0.0 (Scala 2.10)","packageLabel":"spark-image-073c1b52ace74f251fae2680624a0d8d184a8b57096d1c21c5ce56c29be6a37a","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"latest-stable-gpu-scala2.11","displayName":"Latest stable (3.5 LTS, GPU, Scala 2.11)","packageLabel":"spark-image-71e2fd18fdbbb732d6adec03e171846687b6ec85a572e5931e8a9ed9b62e7c32","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.4.x-scala2.11","displayName":"3.4 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-a5615cb1adf0d2305f2b93188c6720174ec3e782d100fcbfa96ff870392861df","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.0.2-db3-scala2.10","displayName":"Spark 2.0.2-db3 (Scala 2.10)","packageLabel":"spark-image-584091dedb690de20e8cf22d9e02fdcce1281edda99eedb441a418d50e28088f","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.2.x-scala2.10","displayName":"3.2 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-557788bea0eea16bbf7a8ba13ace07e64dd7fc86270bd5cea086097fe886431f","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"latest-experimental-scala2.10","displayName":"Latest experimental (Scala 2.10)","packageLabel":"spark-image-ec81b6840af02ee2321dd8dfe2587437bbcddf024d4ae287f326a98fac406a6c","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"4.0.x-gpu-scala2.11","displayName":"4.0 beta (GPU, Scala 2.11)","packageLabel":"spark-image-a3385a64e4dc2345f24e5d055047beda6c310f7b323b5372e382a89519c16080","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.1.0-db1-scala2.11","displayName":"Spark 2.1.0-db1 (Scala 2.11)","packageLabel":"spark-image-e8ad5b72cf0f899dcf2b4720c1f572ab0e87a311d6113b943b4e1d4a7edb77eb","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.1.1-db4-scala2.11","displayName":"Spark 2.1.1-db4 (Scala 2.11)","packageLabel":"spark-image-52bca0ca866e3f4243d3820a783abf3b9b3b553edf234abef14b892657ceaca9","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"latest-rc-scala2.11","displayName":"Latest RC (4.0, Scala 2.11)","packageLabel":"spark-image-0dbee48aa83ff77df1e7b2892d90c671c7b31f94e9fd724e17b9dc4153e12df3","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"latest-stable-scala2.11","displayName":"Latest stable (3.5 LTS, Scala 2.11)","packageLabel":"spark-image-4f5f9fb3a7177ac43f84a20f819e8ad76833e356707ed0b79812a2f837ac0a06","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"2.1.0-db2-scala2.10","displayName":"Spark 2.1.0-db2 (Scala 2.10)","packageLabel":"spark-image-a2ca4f6b58c95f78dca91b1340305ab3fe32673bd894da2fa8e1dc8a9f8d0478","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"1.6.x-ubuntu15.10-hadoop1","displayName":"Spark 1.6.x (Hadoop 1)","packageLabel":"spark-image-8cea23fb9094e174bf5815d79009f4a8e383eb86cf2909cf6c6434ed8da2a16a","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.0.2-db4-scala2.11","displayName":"Spark 2.0.2-db4 (Scala 2.11)","packageLabel":"spark-image-7dbc7583e8271765b8a1508cb9e832768e35489bbde2c4c790bc6766aee2fd7f","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"1.6.1-ubuntu15.10-hadoop1","displayName":"Spark 1.6.1 (Hadoop 1)","packageLabel":"spark-image-21d1cac181b7b8856dd1b4214a3a734f95b5289089349db9d9c926cb87d843db","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.x-gpu-scala2.11","displayName":"Spark 2.0 (Auto-updating, GPU, Scala 2.11 experimental)","packageLabel":"spark-image-968b89f1d0ec32e1ee4dacd04838cae25ef44370a441224177a37980d539d83a","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"1.6.2-ubuntu15.10-hadoop1","displayName":"Spark 1.6.2 (Hadoop 1)","packageLabel":"spark-image-8cea23fb9094e174bf5815d79009f4a8e383eb86cf2909cf6c6434ed8da2a16a","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"next-major-version-scala2.11","displayName":"Next major version (4.0 snapshot, Scala 2.11)","packageLabel":"spark-image-0dbee48aa83ff77df1e7b2892d90c671c7b31f94e9fd724e17b9dc4153e12df3","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"1.6.3-db1-hadoop2-scala2.10","displayName":"Spark 1.6.3-db1 (Hadoop 2, Scala 2.10)","packageLabel":"spark-image-eaa8d9b990015a14e032fb2e2e15be0b8d5af9627cd01d855df728b67969d5d9","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"1.6.3-db2-hadoop1-scala2.10","displayName":"Spark 1.6.3-db2 (Hadoop 1, Scala 2.10)","packageLabel":"spark-image-14112ea0645bea94333a571a150819ce85573cf5541167d905b7e6588645cf3b","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"3.5.x-scala2.10","displayName":"3.5 LTS (includes Apache Spark 2.2.1, Scala 2.10)","packageLabel":"spark-image-fff24a42ff1d42290166196e45860557b34d4af8a5e6e794857bd999865393fb","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"1.6.2-ubuntu15.10-hadoop2","displayName":"Spark 1.6.2 (Hadoop 2)","packageLabel":"spark-image-161245e66d887cd775e23286a54bab0b146143e1289f25bd1732beac454a1561","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"1.6.1-ubuntu15.10-hadoop2","displayName":"Spark 1.6.1 (Hadoop 2)","packageLabel":"spark-image-4cafdf8bc6cba8edad12f441e3b3f0a8ea27da35c896bc8290e16b41fd15496a","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.2-db2-scala2.10","displayName":"Spark 2.0.2-db2 (Scala 2.10)","packageLabel":"spark-image-36d48f22cca7a907538e07df71847dd22aaf84a852c2eeea2dcefe24c681602f","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.x-ubuntu15.10-scala2.11","displayName":"Spark 2.0 (Ubuntu 15.10, Scala 2.11, deprecated)","packageLabel":"spark-image-8e1c50d626a52eac5a6c8129e09ae206ba9890f4523775f77af4ad6d99a64c44","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.x-scala2.10","displayName":"Spark 2.0 (Auto-updating, Scala 2.10)","packageLabel":"spark-image-859e88079f97f58d50e25163b39a1943d1eeac0b6939c5a65faba986477e311a","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.1.1-db4-scala2.10","displayName":"Spark 2.1.1-db4 (Scala 2.10)","packageLabel":"spark-image-c7c0224de396cd1563addc1ae4bca6ba823780b6babe6c3729ddf73008f29ba4","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"latest-rc-scala2.10","displayName":"Latest RC (Scala 2.10)","packageLabel":"spark-image-ec81b6840af02ee2321dd8dfe2587437bbcddf024d4ae287f326a98fac406a6c","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"latest-stable-scala2.10","displayName":"Latest stable (3.5 LTS, Scala 2.10)","packageLabel":"spark-image-fff24a42ff1d42290166196e45860557b34d4af8a5e6e794857bd999865393fb","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"2.0.2-db1-scala2.11","displayName":"Spark 2.0.2-db1 (Scala 2.11)","packageLabel":"spark-image-c2d623f03dd44097493c01aa54a941fc31978ebe6d759b36c75b716b2ff6ab9c","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.2-db4-scala2.10","displayName":"Spark 2.0.2-db4 (Scala 2.10)","packageLabel":"spark-image-859e88079f97f58d50e25163b39a1943d1eeac0b6939c5a65faba986477e311a","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.1.1-db5-scala2.10","displayName":"Spark 2.1.1-db5 (Scala 2.10)","packageLabel":"spark-image-74133df2c13950431298d1cab3e865c191d83ac33648a8590495c52fc644c654","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"3.4.x-gpu-scala2.11","displayName":"3.4 (includes Apache Spark 2.2.0, GPU, Scala 2.11)","packageLabel":"spark-image-613a129fcaa93423a4de06407c9f93e341ed5c6b02d69179d2703c8bb47e2b99","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2 (Hadoop 1, deprecated)","packageLabel":"spark-image-c9d2a8abf41f157a4acc6d52bc721090346f6fea2de356f3a66e388f54481698","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"latest-experimental-gpu-scala2.11","displayName":"Latest experimental (4.1 snapshot, GPU, Scala 2.11)","packageLabel":"spark-image-c4280632bffc04f2b9d8817c4d831814d400c1d2d56920d7b9313ad2d17fdd62","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.2.x-scala2.10","displayName":"3.0 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-d549f2d4a523994ecdf37e531b51d5ec7d8be51534bb0ca5322eaad28ba8f557","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"3.0.x-scala2.11","displayName":"3.0 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-67ab3a06d1e83d5b60df7063245eb419a2e9fe329aeeb7e7d9713332c669bb17","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.0.x-scala2.11","displayName":"Spark 2.0 (Auto-updating, Scala 2.11)","packageLabel":"spark-image-7dbc7583e8271765b8a1508cb9e832768e35489bbde2c4c790bc6766aee2fd7f","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.1.x-scala2.10","displayName":"Spark 2.1 (Auto-updating, Scala 2.10)","packageLabel":"spark-image-177f3f02a6a3432d30068332dc857b9161345bdd2ee8a2d2de05bb05cb4b0f4c","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"3.1.x-scala2.11","displayName":"3.1 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-241fa8b78ee6343242b1756b18076270894385ff40a81172a6fb5eadf66155d3","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.1.0-db3-scala2.10","displayName":"Spark 2.1.0-db3 (Scala 2.10)","packageLabel":"spark-image-25a17d070af155f10c4232dcc6248e36a2eb48c24f8d4fc00f34041b86bd1626","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.0.2-db2-scala2.11","displayName":"Spark 2.0.2-db2 (Scala 2.11)","packageLabel":"spark-image-4fa852ba378e97815083b96c9cada7b962a513ec23554a5fc849f7f1dd8c065a","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"3.1.x-scala2.10","displayName":"3.1 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-7efac6b9a8f2da59cb4f6d0caac46cfcb3f1ebf64c8073498c42d0360f846714","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.3.x-scala2.11","displayName":"3.3 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-0badc3d8dfc8cddd55795d02c0b31c76330cfe687d588414f91278197fbc9416","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"next-major-version-gpu-scala2.11","displayName":"Next major version (4.0 snapshot, GPU, Scala 2.11)","packageLabel":"spark-image-a3385a64e4dc2345f24e5d055047beda6c310f7b323b5372e382a89519c16080","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.5.x-gpu-scala2.11","displayName":"3.5 LTS (includes Apache Spark 2.2.1, GPU, Scala 2.11)","packageLabel":"spark-image-71e2fd18fdbbb732d6adec03e171846687b6ec85a572e5931e8a9ed9b62e7c32","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"1.3.x-ubuntu15.10","displayName":"Spark 1.3.0 (Hadoop 1, deprecated)","packageLabel":"spark-image-40d2842670bc3dc178b14042501847d76171437ccf70613fa397a7a24c48b912","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.0.1-db1-scala2.11","displayName":"Spark 2.0.1-db1 (Scala 2.11)","packageLabel":"spark-image-10ab19f634bbfdb860446c326a9f76dc25bfa87de6403b980566279142a289ea","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.2-db3-scala2.11","displayName":"Spark 2.0.2-db3 (Scala 2.11)","packageLabel":"spark-image-7fd7aaa89d55692e429115ae7eac3b1a1dc4de705d50510995f34306b39c2397","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.1.1-db6-scala2.11","displayName":"Spark 2.1.1-db6 (Scala 2.11)","packageLabel":"spark-image-fdad9ef557700d7a8b6bde86feccbcc3c71d1acdc838b0fd299bd19956b1076e","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"1.6.3-db1-hadoop1-scala2.10","displayName":"Spark 1.6.3-db1 (Hadoop 1, Scala 2.10)","packageLabel":"spark-image-d50af1032799546b8ccbeeb76889a20c819ebc2a0e68ea20920cb30d3895d3ae","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.0.2-db1-scala2.10","displayName":"Spark 2.0.2-db1 (Scala 2.10)","packageLabel":"spark-image-654bdd6e9bad70079491987d853b4b7abf3b736fff099701501acaabe0e75c41","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.0.x-ubuntu15.10","displayName":"Spark 2.0 (Ubuntu 15.10, Scala 2.10, deprecated)","packageLabel":"spark-image-a659f3909d51b38d297b20532fc807ecf708cfb7440ce9b090c406ab0c1e4b7e","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"3.5.x-scala2.11","displayName":"3.5 LTS (includes Apache Spark 2.2.1, Scala 2.11)","packageLabel":"spark-image-4f5f9fb3a7177ac43f84a20f819e8ad76833e356707ed0b79812a2f837ac0a06","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"latest-experimental-scala2.11","displayName":"Latest experimental (4.1 snapshot, Scala 2.11)","packageLabel":"spark-image-cc60a4fe7036aedc8b14f64178c1c28a037001b889a6d97e08bf9d02160b8482","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},{"key":"3.2.x-scala2.11","displayName":"3.2 (includes Apache Spark 2.2.0, Scala 2.11)","packageLabel":"spark-image-5537926238bc55cb6cd76ee0f0789511349abead3781c4780721a845f34b5d4e","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":[]},{"key":"2.0.1-db1-scala2.10","displayName":"Spark 2.0.1-db1 (Scala 2.10)","packageLabel":"spark-image-5a13c2db3091986a4e7363006cc185c5b1108c7761ef5d0218506cf2e6643840","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.1.x-scala2.11","displayName":"Spark 2.1 (Auto-updating, Scala 2.11)","packageLabel":"spark-image-fdad9ef557700d7a8b6bde86feccbcc3c71d1acdc838b0fd299bd19956b1076e","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"2.1.0-db1-scala2.10","displayName":"Spark 2.1.0-db1 (Scala 2.10)","packageLabel":"spark-image-f0ab82a5deb7908e0d159e9af066ba05fb56e1edb35bdad41b7ad2fd62a9b546","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"3.0.x-scala2.10","displayName":"3.0 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-d549f2d4a523994ecdf37e531b51d5ec7d8be51534bb0ca5322eaad28ba8f557","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"1.6.0-ubuntu15.10","displayName":"Spark 1.6.0 (Hadoop 1)","packageLabel":"spark-image-10ef758029b8c7e19cd7f4fb52fff9180d75db92ca071bd94c47f3c1171a7cb5","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"1.6.x-ubuntu15.10-hadoop2","displayName":"Spark 1.6.x (Hadoop 2)","packageLabel":"spark-image-161245e66d887cd775e23286a54bab0b146143e1289f25bd1732beac454a1561","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"2.0.0-ubuntu15.10-scala2.11","displayName":"Spark 2.0.0 (Scala 2.11)","packageLabel":"spark-image-b4ec141e751f201399f8358a82efee202560f7ed05e1a04a2ae8778f6324b909","upgradable":true,"deprecated":true,"customerVisible":false,"capabilities":[]},{"key":"2.1.0-db3-scala2.11","displayName":"Spark 2.1.0-db3 (Scala 2.11)","packageLabel":"spark-image-ccbc6b73f158e2001fc1fb8c827bfdde425d8bd6d65cb7b3269784c28bb72c16","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]},{"key":"latest-rc-gpu-scala2.11","displayName":"Latest RC (4.0, gpu-scala 2.11)","packageLabel":"spark-image-a3385a64e4dc2345f24e5d055047beda6c310f7b323b5372e382a89519c16080","upgradable":true,"deprecated":false,"customerVisible":false,"capabilities":[]},{"key":"3.4.x-scala2.10","displayName":"3.4 (includes Apache Spark 2.2.0, Scala 2.10)","packageLabel":"spark-image-b768d65de82a89fbfabff8ec1d2f279ced527c0ec05e83c3ae0c206d2e97edc0","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION"]}],"enablePresentationMode":false,"enableClearStateAndRunAll":true,"enableTableAclsByTier":false,"enableRestrictedClusterCreation":true,"enableFeedback":true,"enableClusterAutoScaling":false,"enableUserVisibleDefaultTags":true,"defaultNumWorkers":0,"serverContinuationTimeoutMillis":10000,"jobsUnreachableThresholdMillis":60000,"driverStderrFilePrefix":"stderr","enableNotebookRefresh":false,"createTableInNotebookImportedFileLink":{"url":"https://docs.databricks.com/_static/notebooks/data-import/imported-file.html","displayName":"Imported File","workspaceFileName":"Imported File Example"},"accountsOwnerUrl":"https://accounts.cloud.databricks.com/registration.html#login","tableAclsDisabledMap":{"spark.databricks.acl.dfAclsEnabled":"false"},"driverStdoutFilePrefix":"stdout","showDbuPricing":true,"databricksDocsBaseHostname":"docs.databricks.com","defaultNodeTypeToPricingUnitsMap":{"r3.2xlarge":2,"i3.4xlarge":4,"class-node":1,"m4.2xlarge":1.5,"r4.xlarge":1,"m4.4xlarge":3,"Standard_DS5_v2":3,"Standard_D2s_v3":0.5,"Standard_DS14":4,"r4.16xlarge":16,"Standard_DS11":0.5,"Standard_D64s_v3":12,"p2.8xlarge":16,"m4.10xlarge":8,"Standard_D8s_v3":1.5,"Standard_E32s_v3":8,"Standard_DS3":0.75,"Standard_DS2_v2":0.5,"r3.8xlarge":8,"r4.4xlarge":4,"dev-tier-node":1,"Standard_L8s":2,"Standard_E4s_v3":1,"Standard_D3_v2":0.75,"Standard_DS15_v2":5,"Standard_D16s_v3":3,"Standard_D5_v2":3,"Standard_E8s_v3":2,"c3.8xlarge":4,"Standard_E2s_v3":0.5,"Standard_DS3_v2":0.75,"r3.4xlarge":4,"Standard_DS4":1.5,"i2.4xlarge":6,"m4.xlarge":0.75,"r4.8xlarge":8,"Standard_H16":4,"Standard_DS14_v2":4,"r4.large":0.5,"Standard_DS12":1,"development-node":1,"i2.2xlarge":3,"g2.8xlarge":6,"i3.large":0.75,"memory-optimized":1,"m4.large":0.4,"Standard_F4s":0.5,"p2.16xlarge":24,"i3.8xlarge":8,"Standard_D32s_v3":6,"i3.16xlarge":16,"Standard_DS12_v2":1,"Standard_L32s":8,"Standard_D4s_v3":0.75,"Standard_DS13":2,"Standard_DS11_v2":0.5,"Standard_DS13_v2":2,"c3.2xlarge":1,"Standard_L4s":1,"Standard_F16s":2,"c4.2xlarge":1,"Standard_L16s":4,"i2.xlarge":1.5,"Standard_DS2":0.5,"compute-optimized":1,"c4.4xlarge":2,"Standard_D2_v2":0.5,"i3.2xlarge":2,"Standard_E16s_v3":4,"Standard_F8s":1,"c3.4xlarge":2,"g2.2xlarge":1.5,"p2.xlarge":2,"m4.16xlarge":12,"Standard_DS4_v2":1.5,"c4.8xlarge":4,"i3.xlarge":1,"r3.xlarge":1,"r4.2xlarge":2,"i2.8xlarge":12},"tableFilesBaseFolder":"/tables","enableSparkDocsSearch":true,"sparkHistoryServerEnabled":true,"enableClusterAppsUIOnServerless":false,"enableEBSVolumesUI":false,"homePageWelcomeMessage":"Welcome to ","metastoreServiceRowLimit":1000000,"enableIPythonImportExport":true,"enableClusterTagsUIForJobs":true,"enableClusterTagsUI":false,"enableNotebookHistoryDiffing":true,"branch":"2.65.961","accountsLimit":3,"enableSparkEnvironmentVariables":true,"enableX509Authentication":false,"useAADLogin":false,"enableStructuredStreamingNbOptimizations":true,"enableNotebookGitBranching":true,"local":false,"enableNotebookLazyRenderWrapper":false,"enableClusterAutoScalingForJobs":true,"enableStrongPassword":false,"showReleaseNote":true,"displayDefaultContainerMemoryGB":6,"broadenedEditPermission":false,"enableNotebookCommandMode":true,"disableS3TableImport":false,"enableArrayParamsEdit":true,"deploymentMode":"production","useSpotForWorkers":true,"removePasswordInAccountSettings":false,"preferStartTerminatedCluster":false,"enableUserInviteWorkflow":true,"createTableConnectorOptionLinks":[{"url":"https://docs.databricks.com/_static/notebooks/redshift.html","displayName":"Amazon Redshift","workspaceFileName":"Amazon Redshift Example"},{"url":"https://docs.databricks.com/_static/notebooks/structured-streaming-kinesis.html","displayName":"Amazon Kinesis","workspaceFileName":"Amazon Kinesis Example"},{"url":"https://docs.databricks.com/_static/notebooks/data-import/jdbc.html","displayName":"JDBC","workspaceFileName":"JDBC Example"},{"url":"https://docs.databricks.com/_static/notebooks/cassandra.html","displayName":"Cassandra","workspaceFileName":"Cassandra Example"},{"url":"https://docs.databricks.com/_static/notebooks/structured-streaming-etl-kafka.html","displayName":"Kafka","workspaceFileName":"Kafka Example"},{"url":"https://docs.databricks.com/_static/notebooks/redis.html","displayName":"Redis","workspaceFileName":"Redis Example"},{"url":"https://docs.databricks.com/_static/notebooks/elasticsearch.html","displayName":"Elasticsearch","workspaceFileName":"Elasticsearch Example"}],"enableStaticNotebooks":true,"sandboxForUrlSandboxFrame":"allow-scripts allow-popups allow-popups-to-escape-sandbox allow-forms","enableCssTransitions":true,"serverlessEnableElasticDisk":true,"minClusterTagKeyLength":1,"showHomepageFeaturedLinks":true,"pricingURL":"https://databricks.com/product/pricing","enableClusterEdit":true,"enableClusterAclsConfig":false,"useTempS3UrlForTableUpload":false,"notifyLastLogin":false,"enableSshKeyUIByTier":false,"enableCreateClusterOnAttach":true,"defaultAutomatedPricePerDBU":0.2,"enableNotebookGitVersioning":true,"defaultMinWorkers":2,"files":"files/","feedbackEmail":"feedback@databricks.com","enableDriverLogsUI":true,"defaultMaxWorkers":8,"enableWorkspaceAclsConfig":false,"serverlessRunPythonAsLowPrivilegeUser":false,"dropzoneMaxFileSize":2047,"enableNewClustersList":true,"enableNewDashboardViews":true,"enableJobListPermissionFilter":true,"driverLog4jFilePrefix":"log4j","enableSingleSignOn":true,"enableMavenLibraries":true,"displayRowLimit":1000,"deltaProcessingAsyncEnabled":true,"enableSparkEnvironmentVariablesUI":false,"defaultSparkVersion":{"key":"3.5.x-scala2.11","displayName":"3.5 LTS (includes Apache Spark 2.2.1, Scala 2.11)","packageLabel":"spark-image-4f5f9fb3a7177ac43f84a20f819e8ad76833e356707ed0b79812a2f837ac0a06","upgradable":true,"deprecated":false,"customerVisible":true,"capabilities":["SUPPORTS_END_TO_END_ENCRYPTION","SUPPORTS_TABLE_ACLS"]},"deprecatedEnableStructuredDataAcls":false,"enableCustomSpotPricing":false,"enableMountAclsConfig":false,"defaultAutoterminationMin":120,"useDevTierHomePage":true,"disableExportNotebook":false,"enableClusterClone":true,"enableNotebookLineNumbers":true,"enablePublishHub":false,"notebookHubUrl":"http://hub.dev.databricks.com/","showSqlEndpoints":false,"enableNotebookDatasetInfoView":true,"defaultTagKeys":{"CLUSTER_NAME":"ClusterName","VENDOR":"Vendor","CLUSTER_TYPE":"ResourceClass","CREATOR":"Creator","CLUSTER_ID":"ClusterId"},"enableClusterAclsByTier":false,"databricksDocsBaseUrl":"https://docs.databricks.com/","azurePortalLink":"https://portal.azure.com","cloud":"AWS","customSparkVersionPrefix":"custom:","disallowAddingAdmins":true,"enableSparkConfUI":true,"featureTier":"DEVELOPER_BASIC_TIER","mavenCentralSearchEndpoint":"http://search.maven.org/solrsearch/select","defaultServerlessClusterModel":{"cluster_name":"","node_type_id":"i3.2xlarge","spark_version":"latest-stable-scala2.11","num_workers":null,"enable_jdbc_auto_start":true,"custom_tags":{"ResourceClass":"Serverless"},"autoscale":{"min_workers":2,"max_workers":20},"spark_conf":{"spark.databricks.cluster.profile":"serverless","spark.databricks.repl.allowedLanguages":"sql,python,r","spark.databricks.acl.enabled":"false","spark.databricks.acl.sqlOnly":"false"},"aws_attributes":{"ebs_volume_count":null,"availability":"ON_DEMAND","first_on_demand":1,"ebs_volume_type":null,"spot_bid_price_percent":100,"zone_id":"us-west-2c","ebs_volume_size":null},"autotermination_minutes":0,"enable_elastic_disk":false,"default_tags":{"Vendor":"Databricks","Creator":"soumendas0123@gmail.com","ClusterName":null,"ClusterId":"<Generated after creation>"}},"enableOrgSwitcherUI":true,"bitbucketCloudBaseApiV2Url":"https://api.bitbucket.org/2.0","clustersLimit":1,"enableJdbcImport":true,"enableClusterAppsUIOnNormalClusters":false,"enableElasticDisk":false,"logfiles":"logfiles/","enableRelativeNotebookLinks":true,"enableMultiSelect":true,"homePageLogo":"login/databricks_logoTM_rgb_TM.svg","enableWebappSharding":true,"enableNotebookParamsEdit":true,"enableClusterDeltaUpdates":true,"enableSingleSignOnLogin":false,"separateTableForJobClusters":true,"ebsVolumeSizeLimitGB":{"GENERAL_PURPOSE_SSD":[100,4096],"THROUGHPUT_OPTIMIZED_HDD":[500,4096]},"enableMountAcls":false,"requireEmailUserName":true,"enableRServerless":true,"dbcFeedbackURL":"mailto:feedback@databricks.com","enableMountAclService":true,"showVersion":true,"serverlessClustersByDefault":false,"enableWorkspaceAcls":false,"maxClusterTagKeyLength":127,"gitHash":"","clusterTagReservedPrefixes":[],"tableAclsEnabledMap":{"spark.databricks.acl.dfAclsEnabled":"true"},"showWorkspaceFeaturedLinks":true,"signupUrl":"https://databricks.com/try-databricks","databricksDocsNotebookPathPrefix":"^https://docs\\.databricks\\.com/_static/notebooks/.+$","serverlessAttachEbsVolumesByDefault":false,"enableTokensConfig":false,"allowFeedbackForumAccess":true,"enablePythonVersionUI":true,"enableImportFromUrl":true,"allowDisplayHtmlByUrl":true,"enableTokens":false,"enableMiniClusters":true,"enableNewJobList":true,"enableDebugUI":false,"enableStreamingMetricsDashboard":true,"allowNonAdminUsers":true,"enableSingleSignOnByTier":false,"enableJobsRetryOnTimeout":true,"loginLogo":"/login/databricks_logoTM_rgb_TM.svg","useStandardTierUpgradeTooltips":true,"staticNotebookResourceUrl":"https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/","enableSpotClusterType":true,"enableSparkPackages":true,"checkAadUserInWorkspaceTenant":false,"dynamicSparkVersions":true,"useIframeForHtmlResult":false,"enableClusterTagsUIByTier":false,"enableNotebookHistoryUI":true,"addWhitespaceAfterLastNotebookCell":true,"enableClusterLoggingUI":true,"enableDatabaseDropdownInTableUI":true,"showDebugCounters":false,"enableInstanceProfilesUI":false,"enableFolderHtmlExport":true,"homepageFeaturedLinks":[{"linkURI":"https://docs.databricks.com/_static/notebooks/gentle-introduction-to-apache-spark.html","displayName":"Introduction to Apache Spark on Databricks","icon":"img/home/Python_icon.svg"},{"linkURI":"https://docs.databricks.com/_static/notebooks/databricks-for-data-scientists.html","displayName":"Databricks for Data Scientists","icon":"img/home/Scala_icon.svg"},{"linkURI":"https://docs.databricks.com/_static/notebooks/structured-streaming-python.html","displayName":"Introduction to Structured Streaming","icon":"img/home/Python_icon.svg"}],"enableClusterStart":false,"maxImportFileVersion":5,"enableEBSVolumesUIByTier":false,"enableTableAclService":true,"removeSubCommandCodeWhenExport":true,"upgradeURL":"https://accounts.cloud.databricks.com/registration.html#login","maxAutoterminationMinutes":10000,"showResultsFromExternalSearchEngine":true,"autoterminateClustersByDefault":true,"notebookLoadingBackground":"#fff","sshContainerForwardedPort":2200,"enableServerAutoComplete":true,"enableStaticHtmlImport":true,"enableInstanceProfilesByTier":false,"showForgotPasswordLink":true,"defaultMemoryPerContainerMB":6000,"enablePresenceUI":true,"minAutoterminationMinutes":10,"accounts":true,"useOnDemandClustersByDefault":true,"useFramedStaticNotebooks":false,"enableNewProgressReportUI":true,"enableAutoCreateUserUI":true,"defaultCoresPerContainer":4,"showTerminationReason":true,"enableNewClustersGet":true,"showPricePerDBU":false,"showSqlProxyUI":true,"enableNotebookErrorHighlighting":true};</script>
<script>var __DATABRICKS_NOTEBOOK_MODEL = {"version":"NotebookV1","origId":2857777371931835,"name":"My_Notebook","language":"scala","commands":[{"version":"CommandV1","origId":2857777371931836,"guid":"0d3dd720-1013-400e-a260-86776e5b57e9","subtype":"command","commandType":"auto","position":1.0,"command":"val df = sqlContext.read.format(\"com.databricks.spark.csv\").option(\"header\",\"true\").option(\"delimiter\",\";\").option(\"inferSchema\",\"true\").load(\"/FileStore/tables/Project_1_dataset_bank_full-c611b.csv\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">df: org.apache.spark.sql.DataFrame = [age;"job";"marital";"education";"default";"balance";"housing";"loan";"contact";"day";"month";"duration";"campaign";"pdays";"previous";"poutcome";"y": string]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"Cancelled","error":null,"workflows":[],"startTime":1519510861350,"submitTime":1519510861335,"finishTime":1519510862012,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"70fc20b7-48db-4c45-b660-5290e65bb270"},{"version":"CommandV1","origId":3244522131565424,"guid":"9fce474a-fdff-44cf-b5ca-2bc6ca9ba62c","subtype":"command","commandType":"auto","position":2.0,"command":"df.first()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res44: org.apache.spark.sql.Row = [58;"management";"married";"tertiary";"no";2143;"yes";"no";"unknown";5;"may";261;1;-1;0;"unknown";"no"]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:37: error: too many arguments for method first: ()org.apache.spark.sql.Row\n df.first(5)\n ^\n</div>","error":null,"workflows":[],"startTime":1519475879981,"submitTime":1519475879973,"finishTime":1519475880223,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"f3e38bd8-28d0-4c8b-b175-ab8bab4e96e3"},{"version":"CommandV1","origId":3244522131565425,"guid":"6d5bbd22-d757-41d3-a100-db22df9bf2dd","subtype":"command","commandType":"auto","position":3.0,"command":"df.take(5)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res45: Array[org.apache.spark.sql.Row] = Array([58;"management";"married";"tertiary";"no";2143;"yes";"no";"unknown";5;"may";261;1;-1;0;"unknown";"no"], [44;"technician";"single";"secondary";"no";29;"yes";"no";"unknown";5;"may";151;1;-1;0;"unknown";"no"], [33;"entrepreneur";"married";"secondary";"no";2;"yes";"yes";"unknown";5;"may";76;1;-1;0;"unknown";"no"], [47;"blue-collar";"married";"unknown";"no";1506;"yes";"no";"unknown";5;"may";92;1;-1;0;"unknown";"no"], [33;"unknown";"single";"unknown";"no";1;"no";"no";"unknown";5;"may";198;1;-1;0;"unknown";"no"])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475884880,"submitTime":1519475884865,"finishTime":1519475885050,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"a2d097d4-0f81-4318-b30c-e12c4ad5e2f5"},{"version":"CommandV1","origId":572957022278265,"guid":"ad0d2125-293a-4104-ae7f-c9de82d5e9bc","subtype":"command","commandType":"auto","position":3.5,"command":"df.show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+----------------------------------------------------------------------------------------------------------------------------------------------------+\n|age;"job";"marital";"education";"default";"balance";"housing";"loan";"contact";"day";"month";"duration";"campaign";"pdays";"previous";"poutcome";"y"|\n+----------------------------------------------------------------------------------------------------------------------------------------------------+\n| 58;"management";"...|\n| 44;"technician";"...|\n| 33;"entrepreneur"...|\n| 47;"blue-collar";...|\n| 33;"unknown";"sin...|\n| 35;"management";"...|\n| 28;"management";"...|\n| 42;"entrepreneur"...|\n| 58;"retired";"mar...|\n| 43;"technician";"...|\n| 41;"admin.";"divo...|\n| 29;"admin.";"sing...|\n| 53;"technician";"...|\n| 58;"technician";"...|\n| 57;"services";"ma...|\n| 51;"retired";"mar...|\n| 45;"admin.";"sing...|\n| 57;"blue-collar";...|\n| 60;"retired";"mar...|\n| 33;"services";"ma...|\n+----------------------------------------------------------------------------------------------------------------------------------------------------+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475912301,"submitTime":1519475912290,"finishTime":1519475912528,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"e26c2c73-3a56-4363-8f8c-a2862ddae613"},{"version":"CommandV1","origId":572957022278266,"guid":"2049780c-deca-4e71-bc0d-4f5df7d4fe56","subtype":"command","commandType":"auto","position":3.75,"command":"df.show(10)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+----------------------------------------------------------------------------------------------------------------------------------------------------+\n|age;"job";"marital";"education";"default";"balance";"housing";"loan";"contact";"day";"month";"duration";"campaign";"pdays";"previous";"poutcome";"y"|\n+----------------------------------------------------------------------------------------------------------------------------------------------------+\n| 58;"management";"...|\n| 44;"technician";"...|\n| 33;"entrepreneur"...|\n| 47;"blue-collar";...|\n| 33;"unknown";"sin...|\n| 35;"management";"...|\n| 28;"management";"...|\n| 42;"entrepreneur"...|\n| 58;"retired";"mar...|\n| 43;"technician";"...|\n+----------------------------------------------------------------------------------------------------------------------------------------------------+\nonly showing top 10 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475928567,"submitTime":1519475928551,"finishTime":1519475928845,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"8e33e503-432d-41df-b2c4-118a10ce23e8"},{"version":"CommandV1","origId":3244522131565426,"guid":"69bd46ce-ca38-4279-af43-19e876c58608","subtype":"command","commandType":"auto","position":4.0,"command":"df.columns","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res49: Array[String] = Array(age;"job";"marital";"education";"default";"balance";"housing";"loan";"contact";"day";"month";"duration";"campaign";"pdays";"previous";"poutcome";"y")\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475938619,"submitTime":1519475938604,"finishTime":1519475938686,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"6c47a278-3017-4da9-80a7-2bd2ceaf38c3"},{"version":"CommandV1","origId":3244522131565427,"guid":"c8a823a8-dee6-495b-a49d-eedb63722424","subtype":"command","commandType":"auto","position":5.0,"command":"df.printSchema()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">root\n |-- age;"job";"marital";"education";"default";"balance";"housing";"loan";"contact";"day";"month";"duration";"campaign";"pdays";"previous";"poutcome";"y": string (nullable = true)\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475944382,"submitTime":1519475944368,"finishTime":1519475944447,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"889e3602-3193-4249-8fbc-62387b5b975a"},{"version":"CommandV1","origId":3244522131565428,"guid":"b969177a-4680-40b1-82c7-5a1ba441809a","subtype":"command","commandType":"auto","position":6.0,"command":"val dfn =sqlContext.read.format(\"com.databricks.spark.csv\").option(\"header\",\"true\").option(\"delimiter\",\";\").option(\"inferSchema\",\"true\").load(\"/FileStore/tables/new.csv\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">dfn: org.apache.spark.sql.DataFrame = [age: int, job: string, marital: string, education: string, default: string, balance: int, housing: string, loan: string, contact: string, day: int, month: string, duration: int, campaign: int, pdays: int, previous: int, poutcome: string, y: string]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:2: error: ')' expected but ';' found.\nval df = sqlContext.read.format("com.databricks.spark.csv").option("header","true").option("delimiter","";""").option("inferSchema","true").load("/FileStore/tables/Project_1_dataset_bank_full-c611b.csv")\n ^\n</div>","error":null,"workflows":[],"startTime":1519513130083,"submitTime":1519513130072,"finishTime":1519513130891,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"b340c948-db26-4242-9e99-24ad28c13b14"},{"version":"CommandV1","origId":3244522131565429,"guid":"77fc2ad6-f1fe-42df-a667-21690e4d9c03","subtype":"command","commandType":"auto","position":7.0,"command":"dfn.printSchema()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">root\n |-- age: integer (nullable = true)\n |-- job: string (nullable = true)\n |-- marital: string (nullable = true)\n |-- education: string (nullable = true)\n |-- default: string (nullable = true)\n |-- balance: integer (nullable = true)\n |-- housing: string (nullable = true)\n |-- loan: string (nullable = true)\n |-- contact: string (nullable = true)\n |-- day: integer (nullable = true)\n |-- month: string (nullable = true)\n |-- duration: integer (nullable = true)\n |-- campaign: integer (nullable = true)\n |-- pdays: integer (nullable = true)\n |-- previous: integer (nullable = true)\n |-- poutcome: string (nullable = true)\n |-- y: string (nullable = true)\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475424099,"submitTime":1519475424088,"finishTime":1519475424187,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"dc6c4aba-64ca-43d7-af66-4fb29a74001e"},{"version":"CommandV1","origId":3244522131565448,"guid":"358db0d2-c11d-4981-a3d2-97b24e4db717","subtype":"command","commandType":"auto","position":10.0,"command":"dfn.show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n|age| job| marital|education|default|balance|housing|loan|contact|day|month|duration|campaign|pdays|previous|poutcome| y|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n| 58| management| married| tertiary| no| 2143| yes| no|unknown| 5| may| 261| 1| -1| 0| unknown| no|\n| 44| technician| single|secondary| no| 29| yes| no|unknown| 5| may| 151| 1| -1| 0| unknown| no|\n| 33|entrepreneur| married|secondary| no| 2| yes| yes|unknown| 5| may| 76| 1| -1| 0| unknown| no|\n| 47| blue-collar| married| unknown| no| 1506| yes| no|unknown| 5| may| 92| 1| -1| 0| unknown| no|\n| 33| unknown| single| unknown| no| 1| no| no|unknown| 5| may| 198| 1| -1| 0| unknown| no|\n| 35| management| married| tertiary| no| 231| yes| no|unknown| 5| may| 139| 1| -1| 0| unknown| no|\n| 28| management| single| tertiary| no| 447| yes| yes|unknown| 5| may| 217| 1| -1| 0| unknown| no|\n| 42|entrepreneur|divorced| tertiary| yes| 2| yes| no|unknown| 5| may| 380| 1| -1| 0| unknown| no|\n| 58| retired| married| primary| no| 121| yes| no|unknown| 5| may| 50| 1| -1| 0| unknown| no|\n| 43| technician| single|secondary| no| 593| yes| no|unknown| 5| may| 55| 1| -1| 0| unknown| no|\n| 41| admin.|divorced|secondary| no| 270| yes| no|unknown| 5| may| 222| 1| -1| 0| unknown| no|\n| 29| admin.| single|secondary| no| 390| yes| no|unknown| 5| may| 137| 1| -1| 0| unknown| no|\n| 53| technician| married|secondary| no| 6| yes| no|unknown| 5| may| 517| 1| -1| 0| unknown| no|\n| 58| technician| married| unknown| no| 71| yes| no|unknown| 5| may| 71| 1| -1| 0| unknown| no|\n| 57| services| married|secondary| no| 162| yes| no|unknown| 5| may| 174| 1| -1| 0| unknown| no|\n| 51| retired| married| primary| no| 229| yes| no|unknown| 5| may| 353| 1| -1| 0| unknown| no|\n| 45| admin.| single| unknown| no| 13| yes| no|unknown| 5| may| 98| 1| -1| 0| unknown| no|\n| 57| blue-collar| married| primary| no| 52| yes| no|unknown| 5| may| 38| 1| -1| 0| unknown| no|\n| 60| retired| married| primary| no| 60| yes| no|unknown| 5| may| 219| 1| -1| 0| unknown| no|\n| 33| services| married|secondary| no| 0| yes| no|unknown| 5| may| 54| 1| -1| 0| unknown| no|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475976891,"submitTime":1519475976877,"finishTime":1519475977157,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"7e4500cd-eeba-432a-ba82-07aebfae1fd9"},{"version":"CommandV1","origId":3244522131565449,"guid":"d6517eb1-eb76-41ea-aa9c-8d52c49cb7c5","subtype":"command","commandType":"auto","position":11.0,"command":"dfn.show(10)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n|age| job| marital|education|default|balance|housing|loan|contact|day|month|duration|campaign|pdays|previous|poutcome| y|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n| 58| management| married| tertiary| no| 2143| yes| no|unknown| 5| may| 261| 1| -1| 0| unknown| no|\n| 44| technician| single|secondary| no| 29| yes| no|unknown| 5| may| 151| 1| -1| 0| unknown| no|\n| 33|entrepreneur| married|secondary| no| 2| yes| yes|unknown| 5| may| 76| 1| -1| 0| unknown| no|\n| 47| blue-collar| married| unknown| no| 1506| yes| no|unknown| 5| may| 92| 1| -1| 0| unknown| no|\n| 33| unknown| single| unknown| no| 1| no| no|unknown| 5| may| 198| 1| -1| 0| unknown| no|\n| 35| management| married| tertiary| no| 231| yes| no|unknown| 5| may| 139| 1| -1| 0| unknown| no|\n| 28| management| single| tertiary| no| 447| yes| yes|unknown| 5| may| 217| 1| -1| 0| unknown| no|\n| 42|entrepreneur|divorced| tertiary| yes| 2| yes| no|unknown| 5| may| 380| 1| -1| 0| unknown| no|\n| 58| retired| married| primary| no| 121| yes| no|unknown| 5| may| 50| 1| -1| 0| unknown| no|\n| 43| technician| single|secondary| no| 593| yes| no|unknown| 5| may| 55| 1| -1| 0| unknown| no|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\nonly showing top 10 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475967636,"submitTime":1519475967614,"finishTime":1519475967857,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"579ed4f9-b673-497d-bf4a-d301bfea290a"},{"version":"CommandV1","origId":3244522131565450,"guid":"06d215c0-0a27-4e30-a2f5-297012a5eba6","subtype":"command","commandType":"auto","position":12.0,"command":"dfn.select(\"job\").show(100)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-------------+\n| job|\n+-------------+\n| management|\n| technician|\n| entrepreneur|\n| blue-collar|\n| unknown|\n| management|\n| management|\n| entrepreneur|\n| retired|\n| technician|\n| admin.|\n| admin.|\n| technician|\n| technician|\n| services|\n| retired|\n| admin.|\n| blue-collar|\n| retired|\n| services|\n| blue-collar|\n| management|\n| blue-collar|\n| services|\n| retired|\n| admin.|\n| management|\n| entrepreneur|\n| management|\n| technician|\n| technician|\n| management|\n| admin.|\n| blue-collar|\n| management|\n| technician|\n| blue-collar|\n| technician|\n| admin.|\n| admin.|\n| services|\n| management|\n| blue-collar|\n| retired|\n| retired|\n| admin.|\n|self-employed|\n| technician|\n| technician|\n| management|\n| blue-collar|\n| management|\n| management|\n| admin.|\n| technician|\n| entrepreneur|\n| management|\n| blue-collar|\n| blue-collar|\n| services|\n| admin.|\n| technician|\n| blue-collar|\n| services|\n| blue-collar|\n| management|\n| retired|\n| management|\n| technician|\n| blue-collar|\n| blue-collar|\n| services|\n| management|\n| blue-collar|\n| technician|\n| technician|\n| blue-collar|\n| services|\n| services|\n| services|\n| admin.|\n| services|\n| technician|\n| admin.|\n|self-employed|\n| blue-collar|\n| admin.|\n| technician|\n| management|\n| retired|\n| management|\n| technician|\n| admin.|\n| blue-collar|\n| entrepreneur|\n| management|\n| blue-collar|\n| retired|\n| technician|\n| management|\n+-------------+\nonly showing top 100 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475559661,"submitTime":1519475559645,"finishTime":1519475560358,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"3f1f3f73-4342-430f-b786-e7ae63da56d4"},{"version":"CommandV1","origId":3244522131565451,"guid":"38349fa7-2d8d-4a43-a9ef-0e530090d7a2","subtype":"command","commandType":"auto","position":13.0,"command":"dfn.filter(dfn(\"y\")=\"yes\")","commandVersion":0,"state":"error","results":null,"errorSummary":"<div class=\"ansiout\"><console>:37: error: value update is not a member of org.apache.spark.sql.DataFrame\n dfn.filter(dfn("y")="yes")\n ^\n</div>","error":null,"workflows":[],"startTime":1519475587237,"submitTime":1519475587237,"finishTime":1519475587284,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"b8900d09-d587-40b9-aa59-bc8d8540e7a6"},{"version":"CommandV1","origId":3244522131565458,"guid":"659254e4-d6b8-453d-9e37-cfe85dbf087f","subtype":"command","commandType":"auto","position":20.0,"command":"dfn.filter(dfn(\"duration\")>=99).show(20)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n|age| job| marital|education|default|balance|housing|loan|contact|day|month|duration|campaign|pdays|previous|poutcome| y|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n| 58| management| married| tertiary| no| 2143| yes| no|unknown| 5| may| 261| 1| -1| 0| unknown| no|\n| 44| technician| single|secondary| no| 29| yes| no|unknown| 5| may| 151| 1| -1| 0| unknown| no|\n| 33| unknown| single| unknown| no| 1| no| no|unknown| 5| may| 198| 1| -1| 0| unknown| no|\n| 35| management| married| tertiary| no| 231| yes| no|unknown| 5| may| 139| 1| -1| 0| unknown| no|\n| 28| management| single| tertiary| no| 447| yes| yes|unknown| 5| may| 217| 1| -1| 0| unknown| no|\n| 42|entrepreneur|divorced| tertiary| yes| 2| yes| no|unknown| 5| may| 380| 1| -1| 0| unknown| no|\n| 41| admin.|divorced|secondary| no| 270| yes| no|unknown| 5| may| 222| 1| -1| 0| unknown| no|\n| 29| admin.| single|secondary| no| 390| yes| no|unknown| 5| may| 137| 1| -1| 0| unknown| no|\n| 53| technician| married|secondary| no| 6| yes| no|unknown| 5| may| 517| 1| -1| 0| unknown| no|\n| 57| services| married|secondary| no| 162| yes| no|unknown| 5| may| 174| 1| -1| 0| unknown| no|\n| 51| retired| married| primary| no| 229| yes| no|unknown| 5| may| 353| 1| -1| 0| unknown| no|\n| 60| retired| married| primary| no| 60| yes| no|unknown| 5| may| 219| 1| -1| 0| unknown| no|\n| 28| blue-collar| married|secondary| no| 723| yes| yes|unknown| 5| may| 262| 1| -1| 0| unknown| no|\n| 56| management| married| tertiary| no| 779| yes| no|unknown| 5| may| 164| 1| -1| 0| unknown| no|\n| 32| blue-collar| single| primary| no| 23| yes| yes|unknown| 5| may| 160| 1| -1| 0| unknown| no|\n| 25| services| married|secondary| no| 50| yes| no|unknown| 5| may| 342| 1| -1| 0| unknown| no|\n| 40| retired| married| primary| no| 0| yes| yes|unknown| 5| may| 181| 1| -1| 0| unknown| no|\n| 44| admin.| married|secondary| no| -372| yes| no|unknown| 5| may| 172| 1| -1| 0| unknown| no|\n| 39| management| single| tertiary| no| 255| yes| no|unknown| 5| may| 296| 1| -1| 0| unknown| no|\n| 52|entrepreneur| married|secondary| no| 113| yes| yes|unknown| 5| may| 127| 1| -1| 0| unknown| no|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475673733,"submitTime":1519475673722,"finishTime":1519475674338,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"5e0b96e2-bbdc-497c-bc0b-9f4b12ec3fa6"},{"version":"CommandV1","origId":3244522131565459,"guid":"7cb19dd2-61ad-4cb4-b199-a4f2c058660b","subtype":"command","commandType":"auto","position":21.0,"command":"dfn.groupBy(\"age\").count().show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+-----+\n|age|count|\n+---+-----+\n| 31| 1996|\n| 32| 2085|\n| 33| 1972|\n| 34| 1930|\n| 35| 1894|\n| 36| 1806|\n| 37| 1696|\n| 38| 1466|\n| 39| 1487|\n| 40| 1355|\n| 41| 1291|\n| 42| 1242|\n| 43| 1161|\n| 44| 1136|\n| 45| 1216|\n| 46| 1175|\n| 47| 1088|\n| 48| 997|\n| 49| 994|\n| 50| 939|\n+---+-----+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:37: error: value show is not a member of org.apache.spark.sql.GroupedData\n df2.groupBy("age").show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519475683982,"submitTime":1519475683966,"finishTime":1519475685510,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"fc530c5e-b756-4d55-8942-97be7d08a3c0"},{"version":"CommandV1","origId":3244522131565460,"guid":"72463c83-3659-4a99-8081-45cfa0ce121d","subtype":"command","commandType":"auto","position":22.0,"command":"dfn.groupBy(\"job\").count.show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-------------+-----+\n| job|count|\n+-------------+-----+\n| unemployed| 1303|\n| blue-collar| 9732|\n| services| 4154|\n| unknown| 288|\n| housemaid| 1240|\n| entrepreneur| 1487|\n|self-employed| 1579|\n| retired| 2264|\n| admin.| 5171|\n| management| 9458|\n| technician| 7597|\n| student| 938|\n+-------------+-----+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:37: error: value show is not a member of org.apache.spark.sql.GroupedData\n dfn.groupBy("job").show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519476064932,"submitTime":1519476064919,"finishTime":1519476065981,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"e244cae6-3483-4795-8adf-079c5e1dd1b7"},{"version":"CommandV1","origId":3244522131565461,"guid":"39f12efa-abc7-4cdb-a9cc-5d16ebeb4a8a","subtype":"command","commandType":"auto","position":23.0,"command":"dfn.groupBy(\"y\").count().show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+-----+\n| y|count|\n+---+-----+\n| no|39922|\n|yes| 5289|\n+---+-----+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475721286,"submitTime":1519475721271,"finishTime":1519475722303,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"856494e8-87b9-421e-93a4-e38970b3574a"},{"version":"CommandV1","origId":3244522131565462,"guid":"6ee30192-120f-482a-8c2b-8399b570cc6e","subtype":"command","commandType":"auto","position":24.0,"command":"dfn.registerTempTable(\"bank\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\"></div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475753463,"submitTime":1519475753447,"finishTime":1519475753552,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"747f2f3b-b481-40e4-8563-a8b0e0af4254"},{"version":"CommandV1","origId":3244522131565463,"guid":"557112d8-0f54-41a5-a099-84ee114bb790","subtype":"command","commandType":"auto","position":25.0,"command":"sqlContext.sql(\"select * from bank\").show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n|age| job| marital|education|default|balance|housing|loan|contact|day|month|duration|campaign|pdays|previous|poutcome| y|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n| 58| management| married| tertiary| no| 2143| yes| no|unknown| 5| may| 261| 1| -1| 0| unknown| no|\n| 44| technician| single|secondary| no| 29| yes| no|unknown| 5| may| 151| 1| -1| 0| unknown| no|\n| 33|entrepreneur| married|secondary| no| 2| yes| yes|unknown| 5| may| 76| 1| -1| 0| unknown| no|\n| 47| blue-collar| married| unknown| no| 1506| yes| no|unknown| 5| may| 92| 1| -1| 0| unknown| no|\n| 33| unknown| single| unknown| no| 1| no| no|unknown| 5| may| 198| 1| -1| 0| unknown| no|\n| 35| management| married| tertiary| no| 231| yes| no|unknown| 5| may| 139| 1| -1| 0| unknown| no|\n| 28| management| single| tertiary| no| 447| yes| yes|unknown| 5| may| 217| 1| -1| 0| unknown| no|\n| 42|entrepreneur|divorced| tertiary| yes| 2| yes| no|unknown| 5| may| 380| 1| -1| 0| unknown| no|\n| 58| retired| married| primary| no| 121| yes| no|unknown| 5| may| 50| 1| -1| 0| unknown| no|\n| 43| technician| single|secondary| no| 593| yes| no|unknown| 5| may| 55| 1| -1| 0| unknown| no|\n| 41| admin.|divorced|secondary| no| 270| yes| no|unknown| 5| may| 222| 1| -1| 0| unknown| no|\n| 29| admin.| single|secondary| no| 390| yes| no|unknown| 5| may| 137| 1| -1| 0| unknown| no|\n| 53| technician| married|secondary| no| 6| yes| no|unknown| 5| may| 517| 1| -1| 0| unknown| no|\n| 58| technician| married| unknown| no| 71| yes| no|unknown| 5| may| 71| 1| -1| 0| unknown| no|\n| 57| services| married|secondary| no| 162| yes| no|unknown| 5| may| 174| 1| -1| 0| unknown| no|\n| 51| retired| married| primary| no| 229| yes| no|unknown| 5| may| 353| 1| -1| 0| unknown| no|\n| 45| admin.| single| unknown| no| 13| yes| no|unknown| 5| may| 98| 1| -1| 0| unknown| no|\n| 57| blue-collar| married| primary| no| 52| yes| no|unknown| 5| may| 38| 1| -1| 0| unknown| no|\n| 60| retired| married| primary| no| 60| yes| no|unknown| 5| may| 219| 1| -1| 0| unknown| no|\n| 33| services| married|secondary| no| 0| yes| no|unknown| 5| may| 54| 1| -1| 0| unknown| no|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519475759065,"submitTime":1519475759048,"finishTime":1519475759435,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"b1b88d1d-ddcd-400c-b6b8-46a95b062556"},{"version":"CommandV1","origId":3244522131565466,"guid":"6c3b2b18-1168-4c80-aeeb-1c2ff98faffd","subtype":"command","commandType":"auto","position":28.0,"command":"val ag = dfn.select(\"age\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">ag: org.apache.spark.sql.DataFrame = [age: int]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:33: error: not found: value df2\n val df = df2.select("age")\n ^\n</div>","error":null,"workflows":[],"startTime":1519475845661,"submitTime":1519475845650,"finishTime":1519475845742,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"d9fc74d8-3b3c-4327-864a-bd5a9b0327c8"},{"version":"CommandV1","origId":572957022278267,"guid":"fb340021-2b9e-4338-997e-ed805702630c","subtype":"command","commandType":"auto","position":28.5,"command":"ag.show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+\n|age|\n+---+\n| 58|\n| 44|\n| 33|\n| 47|\n| 33|\n| 35|\n| 28|\n| 42|\n| 58|\n| 43|\n| 41|\n| 29|\n| 53|\n| 58|\n| 57|\n| 51|\n| 45|\n| 57|\n| 60|\n| 33|\n+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519476102337,"submitTime":1519476102324,"finishTime":1519476102537,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"e0f871ed-8738-430b-8393-ebec62448e5d"},{"version":"CommandV1","origId":3244522131565468,"guid":"7c64cb3f-f760-4968-9428-df1b6c14a599","subtype":"command","commandType":"auto","position":30.0,"command":"dfn.select(dfn(\"job\"),dfn(\"age\")).show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------------+---+\n| job|age|\n+------------+---+\n| management| 58|\n| technician| 44|\n|entrepreneur| 33|\n| blue-collar| 47|\n| unknown| 33|\n| management| 35|\n| management| 28|\n|entrepreneur| 42|\n| retired| 58|\n| technician| 43|\n| admin.| 41|\n| admin.| 29|\n| technician| 53|\n| technician| 58|\n| services| 57|\n| retired| 51|\n| admin.| 45|\n| blue-collar| 57|\n| retired| 60|\n| services| 33|\n+------------+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:35: error: not found: value df2\n df2.select(df2("job"),df2("age")).show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519476155336,"submitTime":1519476155327,"finishTime":1519476155893,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"a32708fa-582e-475d-9511-44db83d16e80"},{"version":"CommandV1","origId":3244522131565470,"guid":"fcdde71d-3c8f-476e-9471-5253137bd868","subtype":"command","commandType":"auto","position":32.0,"command":"dfn.select(\"job\").show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------------+\n| job|\n+------------+\n| management|\n| technician|\n|entrepreneur|\n| blue-collar|\n| unknown|\n| management|\n| management|\n|entrepreneur|\n| retired|\n| technician|\n| admin.|\n| admin.|\n| technician|\n| technician|\n| services|\n| retired|\n| admin.|\n| blue-collar|\n| retired|\n| services|\n+------------+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519476290695,"submitTime":1519476290688,"finishTime":1519476291023,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"25a758fd-10fa-42ad-9a19-4a45236e3b3e"},{"version":"CommandV1","origId":3244522131565471,"guid":"0bf7cbe4-6b49-42fd-bbe1-d27151174748","subtype":"command","commandType":"auto","position":33.0,"command":"dfn.filter(dfn(\"age\")>40).show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n|age| job| marital|education|default|balance|housing|loan|contact|day|month|duration|campaign|pdays|previous|poutcome| y|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\n| 58| management| married| tertiary| no| 2143| yes| no|unknown| 5| may| 261| 1| -1| 0| unknown| no|\n| 44| technician| single|secondary| no| 29| yes| no|unknown| 5| may| 151| 1| -1| 0| unknown| no|\n| 47| blue-collar| married| unknown| no| 1506| yes| no|unknown| 5| may| 92| 1| -1| 0| unknown| no|\n| 42|entrepreneur|divorced| tertiary| yes| 2| yes| no|unknown| 5| may| 380| 1| -1| 0| unknown| no|\n| 58| retired| married| primary| no| 121| yes| no|unknown| 5| may| 50| 1| -1| 0| unknown| no|\n| 43| technician| single|secondary| no| 593| yes| no|unknown| 5| may| 55| 1| -1| 0| unknown| no|\n| 41| admin.|divorced|secondary| no| 270| yes| no|unknown| 5| may| 222| 1| -1| 0| unknown| no|\n| 53| technician| married|secondary| no| 6| yes| no|unknown| 5| may| 517| 1| -1| 0| unknown| no|\n| 58| technician| married| unknown| no| 71| yes| no|unknown| 5| may| 71| 1| -1| 0| unknown| no|\n| 57| services| married|secondary| no| 162| yes| no|unknown| 5| may| 174| 1| -1| 0| unknown| no|\n| 51| retired| married| primary| no| 229| yes| no|unknown| 5| may| 353| 1| -1| 0| unknown| no|\n| 45| admin.| single| unknown| no| 13| yes| no|unknown| 5| may| 98| 1| -1| 0| unknown| no|\n| 57| blue-collar| married| primary| no| 52| yes| no|unknown| 5| may| 38| 1| -1| 0| unknown| no|\n| 60| retired| married| primary| no| 60| yes| no|unknown| 5| may| 219| 1| -1| 0| unknown| no|\n| 56| management| married| tertiary| no| 779| yes| no|unknown| 5| may| 164| 1| -1| 0| unknown| no|\n| 44| admin.| married|secondary| no| -372| yes| no|unknown| 5| may| 172| 1| -1| 0| unknown| no|\n| 52|entrepreneur| married|secondary| no| 113| yes| yes|unknown| 5| may| 127| 1| -1| 0| unknown| no|\n| 46| management| single|secondary| no| -246| yes| no|unknown| 5| may| 255| 2| -1| 0| unknown| no|\n| 57| technician| married|secondary| no| 839| no| yes|unknown| 5| may| 225| 1| -1| 0| unknown| no|\n| 49| management| married| tertiary| no| 378| yes| no|unknown| 5| may| 230| 1| -1| 0| unknown| no|\n+---+------------+--------+---------+-------+-------+-------+----+-------+---+-----+--------+--------+-----+--------+--------+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:37: error: type mismatch;\n found : Int(40)\n required: String\n dfn.filter(("age")>40).show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519476336582,"submitTime":1519476336574,"finishTime":1519476336809,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"95c7c663-2336-4d44-87dc-c7035b804c1e"},{"version":"CommandV1","origId":3244522131565473,"guid":"a20198ff-cc1b-4dfa-b3ad-76c17cdc1bb0","subtype":"command","commandType":"auto","position":35.0,"command":"dfn.select(dfn(\"age\")).show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+\n|age|\n+---+\n| 58|\n| 44|\n| 33|\n| 47|\n| 33|\n| 35|\n| 28|\n| 42|\n| 58|\n| 43|\n| 41|\n| 29|\n| 53|\n| 58|\n| 57|\n| 51|\n| 45|\n| 57|\n| 60|\n| 33|\n+---+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:2: error: identifier expected but '(' found.\n dfn.select(dfn.("age")).show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519476399070,"submitTime":1519476399063,"finishTime":1519476399292,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"f10cee00-3e6c-4889-9300-49e5d637d789"},{"version":"CommandV1","origId":3244522131565474,"guid":"74cda999-d48b-4e22-a31b-fddb71e8d123","subtype":"command","commandType":"auto","position":36.0,"command":"dfn.select(dfn(\"age\")+2).show()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---------+\n|(age + 2)|\n+---------+\n| 60|\n| 46|\n| 35|\n| 49|\n| 35|\n| 37|\n| 30|\n| 44|\n| 60|\n| 45|\n| 43|\n| 31|\n| 55|\n| 60|\n| 59|\n| 53|\n| 47|\n| 59|\n| 62|\n| 35|\n+---------+\nonly showing top 20 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519476424024,"submitTime":1519476424012,"finishTime":1519476424643,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"f7657c96-27f8-4486-925a-a3dfe4ec8d6e"},{"version":"CommandV1","origId":2873309271916278,"guid":"fb579188-11e7-43ae-9971-c4fffd1c6403","subtype":"command","commandType":"auto","position":40.0,"command":"val total=dfn.count()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">total: Long = 45211\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:39: error: value show is not a member of Long\n df.show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519476505179,"submitTime":1519476505167,"finishTime":1519476505713,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"53742bc7-1c94-4062-81c2-57af11512647"},{"version":"CommandV1","origId":1386141370648011,"guid":"adb0de97-b2ba-4442-bdee-7f42f2909a6f","subtype":"command","commandType":"auto","position":42.0,"command":"print(total)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">45211</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:39: error: value take is not a member of Long\n total.take()\n ^\n</div>","error":null,"workflows":[],"startTime":1519481455775,"submitTime":1519481455758,"finishTime":1519481455860,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"c693510a-01c8-404f-b1b8-b0530189e73d"},{"version":"CommandV1","origId":1742845683372380,"guid":"17dbec95-7d23-4e1a-aa31-771afe4e464c","subtype":"command","commandType":"auto","position":43.0,"command":"dfn.groupBy(\"y\").count.show().map(x => )","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+---+-----+\n| y|count|\n+---+-----+\n| no|39922|\n|yes| 5289|\n+---+-----+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:35: error: not found: value dfn\n dfn.groupBy("age").count.show()\n ^\n</div>","error":null,"workflows":[],"startTime":1519512351694,"submitTime":1519512351680,"finishTime":1519512354235,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"d3351c82-8a1e-41ba-b782-8f5a377049c8"},{"version":"CommandV1","origId":402049653844174,"guid":"8e06e79e-cca9-4d60-9668-c3e4ba74b2c5","subtype":"command","commandType":"auto","position":44.0,"command":"val sqlContext= new org.apache.spark.sql.SQLContext(sc)","commandVersion":0,"state":"error","results":{"type":"html","data":"<div class=\"ansiout\"></div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"org.apache.spark.SparkException: In Databricks, developers should utilize the shared HiveContext instead of creating one using the constructor. In Scala and Python notebooks, the shared context can be accessed as sqlContext. When running a job, you can access the shared context by calling SQLContext.getOrCreate(SparkContext.getOrCreate()).","error":"<div class=\"ansiout\">\tat org.apache.spark.sql.SQLContext.<init>(SQLContext.scala:98)\n\tat org.apache.spark.sql.SQLContext.<init>(SQLContext.scala:77)</div>","workflows":[],"startTime":1519511540564,"submitTime":1519511540564,"finishTime":1519511540723,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"e465d64d-7bb9-411b-9153-5a361b92a315"},{"version":"CommandV1","origId":402049653844175,"guid":"6ef7fdea-1765-4f9c-9d6d-870d92a6f242","subtype":"command","commandType":"auto","position":45.0,"command":"import sqlContext.implicits._","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import sqlContext.implicits._\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519508525160,"submitTime":1519508525141,"finishTime":1519508525338,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"8f08faf1-4737-42af-94ca-188636da5d24"},{"version":"CommandV1","origId":402049653844177,"guid":"bea926ec-e719-455d-acb9-3c17cadc25b2","subtype":"command","commandType":"auto","position":47.0,"command":"case class Project(age: Int,job: String,marital: String,education: String,default: String,balance: Int,housing: String,loan: String,contact: String,day :Int,month: String,duration: Int,campaign: Int,pdays: Int,previous: Int,poutcome: String,y : String)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">defined class Project\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519508611451,"submitTime":1519508611046,"finishTime":1519508619623,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"6d36d3a1-cbf4-49f0-b311-1f06e02c8da9"},{"version":"CommandV1","origId":402049653844178,"guid":"791df881-d680-434a-a738-397c938fcd8c","subtype":"command","commandType":"auto","position":48.0,"command":"val bank = sc.textFile(\"/FileStore/tables/new.csv\").map(_.split(\";\")).map(p => Project(p(0).trim.toInt,p(1),p(2),p(3),p(4),p(5).trim.toInt,p(6),p(7),p(8),p(9).trim.toInt,p(10),p(11).trim.toInt,p(12).trim.toInt,p(13).trim.toInt,p(14).trim.toInt,p(15),p(16))).toDF()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">bank: org.apache.spark.sql.DataFrame = [age: int, job: string, marital: string, education: string, default: string, balance: int, housing: string, loan: string, contact: string, day: int, month: string, duration: int, campaign: int, pdays: int, previous: int, poutcome: string, y: string]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:2: error: illegal start of simple expression\n val bank = sc.textFile("/FileStore/tables/new.csv").map(_.split(";")).map(p => Project(p(0).trim.toInt,p(1),p(2),p(3),p(4),p(5).trim.toInt,p(6),p(7),p(8),p(9).trim.toInt,p(10),p(11).trim.toInt,p(12).trim.toInt,p(13).trim.toInt,p(14).trim.toInt,p(15),p(16),)).toDF()\n ^\n</div>","error":null,"workflows":[],"startTime":1519508624067,"submitTime":1519508624043,"finishTime":1519508627070,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"f6f88684-e713-4f05-b488-6dc170c1e8bd"},{"version":"CommandV1","origId":402049653844179,"guid":"6efbc44f-3fab-4de5-a22d-82900b730c7c","subtype":"command","commandType":"auto","position":49.0,"command":"bank.printSchema()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">root\n |-- age: integer (nullable = false)\n |-- job: string (nullable = true)\n |-- marital: string (nullable = true)\n |-- education: string (nullable = true)\n |-- default: string (nullable = true)\n |-- balance: integer (nullable = false)\n |-- housing: string (nullable = true)\n |-- loan: string (nullable = true)\n |-- contact: string (nullable = true)\n |-- day: integer (nullable = false)\n |-- month: string (nullable = true)\n |-- duration: integer (nullable = false)\n |-- campaign: integer (nullable = false)\n |-- pdays: integer (nullable = false)\n |-- previous: integer (nullable = false)\n |-- poutcome: string (nullable = true)\n |-- y: string (nullable = true)\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519508639415,"submitTime":1519508639393,"finishTime":1519508639615,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"5e3f5c7b-0cbe-4dfb-9b97-3a3e18960173"},{"version":"CommandV1","origId":402049653844180,"guid":"12047a32-9239-4456-999e-4776daf2177d","subtype":"command","commandType":"auto","position":50.0,"command":"dfn.printSchema()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">root\n |-- age: integer (nullable = true)\n |-- job: string (nullable = true)\n |-- marital: string (nullable = true)\n |-- education: string (nullable = true)\n |-- default: string (nullable = true)\n |-- balance: integer (nullable = true)\n |-- housing: string (nullable = true)\n |-- loan: string (nullable = true)\n |-- contact: string (nullable = true)\n |-- day: integer (nullable = true)\n |-- month: string (nullable = true)\n |-- duration: integer (nullable = true)\n |-- campaign: integer (nullable = true)\n |-- pdays: integer (nullable = true)\n |-- previous: integer (nullable = true)\n |-- poutcome: string (nullable = true)\n |-- y: string (nullable = true)\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"<div class=\"ansiout\"><console>:35: error: not found: value dfn\n dfn.printSchema()\n ^\n</div>","error":null,"workflows":[],"startTime":1519508708017,"submitTime":1519508707996,"finishTime":1519508708196,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"ab46c777-a113-4224-8304-201da8c5852e"},{"version":"CommandV1","origId":402049653844182,"guid":"da3ea20e-edab-4124-afd3-80dcf5ee05aa","subtype":"command","commandType":"auto","position":51.0,"command":"bank.show()","commandVersion":0,"state":"error","results":{"type":"html","data":"<div class=\"ansiout\"></div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":"org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 2.0 failed 1 times, most recent failure: Lost task 0.0 in stage 2.0 (TID 3, localhost): java.lang.NumberFormatException: For input string: \"age\"","error":"<div class=\"ansiout\">\tat java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)\n\tat java.lang.Integer.parseInt(Integer.java:580)\n\tat java.lang.Integer.parseInt(Integer.java:615)\n\tat scala.collection.immutable.StringLike$class.toInt(StringLike.scala:229)\n\tat scala.collection.immutable.StringOps.toInt(StringOps.scala:31)\n\tat line2fcae6ae98554d048908ce4d43c5bbe814.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$2.apply(<console>:37)\n\tat line2fcae6ae98554d048908ce4d43c5bbe814.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$2.apply(<console>:37)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$10.next(Iterator.scala:312)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:727)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1157)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1157)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1157)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$5.apply(SparkPlan.scala:212)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$5.apply(SparkPlan.scala:212)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1860)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1860)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:72)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:96)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:235)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1431)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1419)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1834)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1847)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1860)\n\tat org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:212)\n\tat org.apache.spark.sql.execution.Limit.executeCollect(basicOperators.scala:165)\n\tat org.apache.spark.sql.execution.SparkPlan.executeCollectPublic(SparkPlan.scala:174)\n\tat org.apache.spark.sql.DataFrame$$anonfun$org$apache$spark$sql$DataFrame$$execute$1$1.apply(DataFrame.scala:1500)\n\tat org.apache.spark.sql.DataFrame$$anonfun$org$apache$spark$sql$DataFrame$$execute$1$1.apply(DataFrame.scala:1500)\n\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:56)\n\tat org.apache.spark.sql.DataFrame.withNewExecutionId(DataFrame.scala:2092)\n\tat org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$execute$1(DataFrame.scala:1499)\n\tat org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$collect(DataFrame.scala:1506)\n\tat org.apache.spark.sql.DataFrame$$anonfun$head$1.apply(DataFrame.scala:1376)\n\tat org.apache.spark.sql.DataFrame$$anonfun$head$1.apply(DataFrame.scala:1375)\n\tat org.apache.spark.sql.DataFrame.withCallback(DataFrame.scala:2105)\n\tat org.apache.spark.sql.DataFrame.head(DataFrame.scala:1375)\n\tat org.apache.spark.sql.DataFrame.take(DataFrame.scala:1457)\n\tat org.apache.spark.sql.DataFrame.showString(DataFrame.scala:170)\n\tat org.apache.spark.sql.DataFrame.show(DataFrame.scala:350)\n\tat org.apache.spark.sql.DataFrame.show(DataFrame.scala:311)\n\tat org.apache.spark.sql.DataFrame.show(DataFrame.scala:319)\nCaused by: java.lang.NumberFormatException: For input string: "age"\n\tat java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)\n\tat java.lang.Integer.parseInt(Integer.java:580)\n\tat java.lang.Integer.parseInt(Integer.java:615)\n\tat scala.collection.immutable.StringLike$class.toInt(StringLike.scala:229)\n\tat scala.collection.immutable.StringOps.toInt(StringOps.scala:31)\n\tat Notebook$$anonfun$2.apply(<console>:37)\n\tat Notebook$$anonfun$2.apply(<console>:37)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$10.next(Iterator.scala:312)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:727)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1157)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1157)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1157)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$5.apply(SparkPlan.scala:212)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$5.apply(SparkPlan.scala:212)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1860)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1860)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:72)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:96)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:235)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)</div>","workflows":[],"startTime":1519508786957,"submitTime":1519508786957,"finishTime":1519508787945,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"c94b74ae-7747-4d16-b1e1-cf1cc397c334"},{"version":"CommandV1","origId":402049653844183,"guid":"0d97d8dc-b93c-41e7-810d-b68d4cd5e5e8","subtype":"command","commandType":"auto","position":52.0,"command":"case class Project(age: Int, job: String, marital: String, education: String, default: String, balance: Int, housing: String, loan: String, contact: String, day :Int, month: String, duration: Int, campaign: Int, pdays: Int, previous: Int, poutcome: String, y : String)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">defined class Project\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519510012401,"submitTime":1519510012377,"finishTime":1519510012792,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"d790c933-8c59-45ca-a289-aa9dfa3b82d1"},{"version":"CommandV1","origId":402049653844184,"guid":"ca7bdc1a-1428-46f8-bf75-b9892c39cfbe","subtype":"command","commandType":"auto","position":53.0,"command":"val rawRDD = sc.textFile(\"/FileStore/tables/new.csv\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">rawRDD: org.apache.spark.rdd.RDD[String] = /FileStore/tables/new.csv MapPartitionsRDD[13] at textFile at <console>:35\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[],"datasetInfos":[]},"errorSummary":null,"error":null,"workflows":[],"startTime":1519510035491,"submitTime":1519510035471,"finishTime":1519510035646,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"afc2f7c3-0de9-49c3-ad40-ed0047d08bdc"},{"version":"CommandV1","origId":402049653844185,"guid":"cf00dae7-c798-42d4-b394-cdb837b2b656","subtype":"command","commandType":"auto","position":54.0,"command":"val proRDD = rawRDD.map{ raw_line => \n val columns = raw_line.split(\";\")\n \n Product(columns(0).toInt,columns(1),columns(2),columns(3),columns(4),columns(5).toInt,columns(6),columns(7),columns(8),columns(9).toInt,columns(10),columns(11).toInt,columns(12).toInt,columns(13).toInt,columns(14).toInt,columns(15),columns(16))\n \n } ","commandVersion":0,"state":"error","results":null,"errorSummary":"<div class=\"ansiout\"><console>:40: error: not found: value Product\n Product(columns(0).toInt,columns(1),columns(2),columns(3),columns(4),columns(5).toInt,columns(6),columns(7),columns(8),columns(9).toInt,columns(10),columns(11).toInt,columns(12).toInt,columns(13).toInt,columns(14).toInt,columns(15),columns(16))\n ^\n</div>","error":null,"workflows":[],"startTime":1519510435372,"submitTime":1519510435372,"finishTime":1519510435447,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"b18a29d9-bb39-46e0-9331-efb3432efb05"},{"version":"CommandV1","origId":402049653844186,"guid":"f135b041-030e-41f1-ae85-355a57875832","subtype":"command","commandType":"auto","position":55.0,"command":"val dfn =sqlContext.read.csv(\"/FileStore/tables/new.csv\").option(\"header\",\"true\").option(\"delimiter\",\";\").option(\"inferSchema\",\"true\")","commandVersion":0,"state":"error","results":null,"errorSummary":"<div class=\"ansiout\"><console>:33: error: value csv is not a member of org.apache.spark.sql.DataFrameReader\n val dfn =sqlContext.read.csv("/FileStore/tables/new.csv").option("header","true").option("delimiter",";").option("inferSchema","true")\n ^\n</div>","error":null,"workflows":[],"startTime":1519513200918,"submitTime":1519513200918,"finishTime":1519513200975,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"streamStates":{},"nuid":"7bb7c225-3890-49e1-b55d-d90fc4746d2a"}],"dashboards":[],"guid":"cbe19ba7-c6d8-4793-9d54-bd2b8622b5f1","globalVars":{},"iPythonMetadata":null,"inputWidgets":{}};</script>
<script
src="https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/js/notebook-main.js"
onerror="window.mainJsLoadError = true;"></script>
</head>
<body>
<script>
if (window.mainJsLoadError) {
var u = 'https://databricks-prod-cloudfront.cloud.databricks.com/static/f12a05aa7eae95a2da8fd637b50b850a1bb31a04ce331f5b43445dd4cda0ff19/js/notebook-main.js';
var b = document.getElementsByTagName('body')[0];
var c = document.createElement('div');
c.innerHTML = ('<h1>Network Error</h1>' +
'<p><b>Please check your network connection and try again.</b></p>' +
'<p>Could not load a required resource: ' + u + '</p>');
c.style.margin = '30px';
c.style.padding = '20px 50px';
c.style.backgroundColor = '#f5f5f5';
c.style.borderRadius = '5px';
b.appendChild(c);
}
</script>
</body>
</html>