python-sdk/tutorials/dataset-uploads/upload_directory_create_file_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1633030069130
    }
   },
   "outputs": [],
   "source": [
    "# Check core SDK version number\n",
    "import azureml.core\n",
    "\n",
    "print(\"Core version:\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Connect to workspace via config.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1633030076765
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from azureml.core import Workspace\n",
    "\n",
    "workspace = Workspace.from_config()\n",
    "print(workspace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uploading to Regular Datastore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload files to datastore and create a file dataset\n",
    "\n",
    "```python\n",
    "    def upload_directory(src_dir, target, pattern=None, overwrite=False, show_progress=True):\n",
    "        \"\"\"Upload source directory to target datastore and create a file dataset\n",
    "\n",
    "        :param src_dir: The local directory to upload.\n",
    "        :type src_dir: str\n",
    "        :param target: Required, the datastore path where the files will be uploaded to.\n",
    "        :type target: azureml.data.datapath.DataPath, azureml.core.datastore.Datastore\n",
    "            or tuple(azureml.core.datastore.Datastore, str) object\n",
    "        :param pattern: Optional, If provided, will filter all the path names matching the given pattern,\n",
    "            similar to Python glob package, supporting '*', '?', and character ranges expressed with [].\n",
    "        :type pattern: str\n",
    "        :param show_progress: Optional,\n",
    "            indicates whether to show progress of the upload in the console. Defaults to be True.\n",
    "        :type show_progress: bool\n",
    "        :return: The created file dataset.\n",
    "        :rtype: azureml.data.FileDataset\n",
    "        \"\"\"\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1633030213010
    }
   },
   "outputs": [],
   "source": [
    "# Syntax:\n",
    "# Dataset.File.upload_directory(src_dir='<your source folder for uploading>/',\n",
    "#            target=DataPath(<datastore>, '<upload path on the datastore>'),\n",
    "#            show_progress=True)\n",
    "# Using workspace default datastore for uploading Pandas dataframe\n",
    "from azureml.core import Dataset\n",
    "from azureml.data.datapath import DataPath\n",
    "\n",
    "# Getting workspace default datastore\n",
    "datastore = workspace.get_default_datastore()\n",
    "# Uploading local directory and creating a FileDataset\n",
    "dataset = Dataset.File.upload_directory(\n",
    "    src_dir=\"./data/\", target=DataPath(datastore, \"/data/\"), show_progress=True\n",
    ")\n",
    "# You can register the datastore created by uploading the directory\n",
    "dataset.register(\n",
    "    workspace=workspace,\n",
    "    name=\"ds_from_directory\",\n",
    "    description=\"This dataset was creating by uploading a directory\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1633030265052
    }
   },
   "outputs": [],
   "source": [
    "# download files back to local\n",
    "dataset.download(\"./downloads\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Upload files to datastore matching a pattern and create a file dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1633030418459
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Upload source files based on their path name using Pattern\n",
    "# In this example, we are going to use only .csv files\n",
    "dataset_pattern = Dataset.File.upload_directory(\n",
    "    src_dir=\"./data/\",\n",
    "    target=DataPath(datastore, \"/data_pattern/\"),\n",
    "    pattern=\"*.csv\",\n",
    "    show_progress=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uploading to credential-less datastore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Register Azure Data Lake Storage Gen1 (ADLS Gen1) as credential-less datastore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "#### Note\n",
    "Following code examples are shown as markdown cells. To run these examples:\n",
    "- Remove ``` at the beginning and end of the cells.\n",
    "- Convert these markdown cells to code cells.\n",
    "- Enter appropriate values inside the '< >' to successfully execute the sample codes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "```\n",
    "from azureml.core import Datastore\n",
    "adlsgen1_datastore_name = 'adlgen1credlessstore' # Datastore name\n",
    "store_name = '<your ADLS store name>' # ADLS Gen1 storage account name\n",
    "subscription_id = '<your subscription id>' # subscription for the ADLS Gen1 storage account\n",
    "resource_group = '<your resource group>' # resource group for the ADLS Gen1 storage account\n",
    "\n",
    "adls_datastore_cred_less = Datastore.register_azure_data_lake(\n",
    "   workspace=ws,\n",
    "   datastore_name=adlsgen1_datastore_name, # Datastore name\n",
    "   subscription_id=subscription_id, # subscription id for the ADLS Gen1 storage account\n",
    "   resource_group=resource_group, # resource group for the ADLS Gen1 storage account\n",
    "   store_name=store_name) # ADLS Gen1 storage account name\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload files to cred-less datastore and create a file dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "```\n",
    "dataset_credless = Dataset.File.upload_directory(src_dir='<your source folder for uploading>/',\n",
    "                    target=DataPath(adls_datastore_cred_less, '<upload path on the datastore>'),\n",
    "                    overwrite=True,\n",
    "                    show_progress=True)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uploading to credential-less datastore using MSI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create or Attach existing compute resource"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "If using an existing compute target, please make sure the compute target has an identity attached, or type a new name to let the below script create a new one."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "```\n",
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "# choose a name for your cluster\n",
    "cluster_name = '<your compute cluster name>'\n",
    "\n",
    "try:\n",
    "    compute_target = ComputeTarget(workspace=workspace, name=cluster_name)\n",
    "    print('Found existing compute target')\n",
    "except ComputeTargetException:\n",
    "    print('Creating a new compute target...')\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', \n",
    "                                                           max_nodes=4,\n",
    "                                                           identity_type='SystemAssigned')\n",
    "\n",
    "    # create the cluster\n",
    "    compute_target = ComputeTarget.create(workspace, cluster_name, compute_config)\n",
    "\n",
    "    # can poll for a minimum number of nodes and for a specific timeout. \n",
    "    # if no min node count is provided it uses the scale settings for the cluster\n",
    "    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
    "\n",
    "# use get_status() to get a detailed status for the current cluster. \n",
    "print(compute_target.get_status().serialize())\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "Make sure that you grant access to the compute cluster created here in the Azure Data Lake Store Gen1 account (not only the storage account, but also the specific folder to access):\n",
    "- Go to the Azure Data Lake Store Gen1 portal, click on \"Data explorer\".\n",
    "- Click on \"Access\" then click \"+\".\n",
    "- In the opened page, click \"Select\" and search for below content:\n",
    "  - \"{your workspace name}/computes/{the compute cluster name created above}\" </dd>\n",
    "- Grant all needed permissions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the remote run script"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "Please replace credential-less datastore name created above and the datapath where files are uploaded."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "```\n",
    "%%writefile data/run.py\n",
    "def run():\n",
    "    from azureml.core import Dataset, Datastore\n",
    "    from azureml.data.datapath import DataPath\n",
    "    from azureml.core.run import Run\n",
    "    import glob\n",
    "\n",
    "    ws = Run.get_context().experiment.workspace\n",
    "    print('Got workspace')\n",
    "    print(ws)\n",
    "    print('Getting datastore')\n",
    "    dstore = Datastore.get(ws, 'adlgen1credlessstore')\n",
    "    print('Got datastore')\n",
    "    print(dstore)\n",
    "    datapath = DataPath(dstore, '/%s/' % 'dataset_from_compute')\n",
    "    print(datapath)\n",
    "    saved_dataset = Dataset.File.upload_directory(\n",
    "        src_dir='./',\n",
    "        target=datapath,\n",
    "        overwrite=True,\n",
    "        show_progress=True)\n",
    "    print(saved_dataset)\n",
    "\n",
    "run()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload files from remote context to target credential-less datastore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "```\n",
    "from azureml.core import ScriptRunConfig, RunConfiguration, Experiment\n",
    "\n",
    "rc = RunConfiguration()\n",
    "rc.target = compute_target\n",
    "\n",
    "# create or load an experiment\n",
    "experiment = Experiment(workspace=workspace, name='MyUploadingExperiment')\n",
    "# run a trial from the train.py code in your current directory\n",
    "config = ScriptRunConfig(source_directory='data', script='run.py', run_config=rc)\n",
    "run = experiment.submit(config)\n",
    "run.wait_for_completion()\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernel_info": {
   "name": "python3"
  },
  "kernelspec": {
   "display_name": "Python 3.8 - AzureML",
   "language": "python",
   "name": "python38-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  },
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}