Example_1a S3 access to AODN S3 Bucket.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demonstrate the usage of Xarray with s3fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imos-data/error.html\n",
      "imos-data/index.html\n",
      "imos-data/robots.txt\n",
      "imos-data/Bureau_of_Meteorology\n",
      "imos-data/CSIRO\n",
      "imos-data/Deakin_University\n",
      "imos-data/Department_of_Defence\n",
      "imos-data/Department_of_Environment_and_Science-Queensland\n",
      "imos-data/Department_of_Transport-Western_Australia\n",
      "imos-data/Derwent_Estuary_Program\n",
      "imos-data/Future_Reef_MAP\n",
      "imos-data/IMOS\n",
      "imos-data/Macquarie_University\n",
      "imos-data/NSW-OEH\n",
      "imos-data/TERN\n",
      "imos-data/UNSW\n",
      "imos-data/UWA\n",
      "imos-data/WAMSI\n"
     ]
    }
   ],
   "source": [
    "import s3fs\n",
    "fs = s3fs.S3FileSystem(anon=True)\n",
    "aodn = fs.ls('imos-data')\n",
    "for item in aodn:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/02',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/03',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/04',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/05',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/06',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/07',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/08',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/09',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/10',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/11',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/12']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import s3fs\n",
    "fs = s3fs.S3FileSystem(anon=True)\n",
    "months = fs.ls('imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014')\n",
    "months"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140101T000000Z.aust.K_490.nc',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140102T000000Z.aust.K_490.nc',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140103T000000Z.aust.K_490.nc',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140104T000000Z.aust.K_490.nc',\n",
       " 'imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140105T000000Z.aust.K_490.nc']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_files = []\n",
    "for month in months:\n",
    "    this_month = [f for f in fs.ls(month) if 'K_490' in f]\n",
    "    all_files=all_files+this_month\n",
    "all_files[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open a NetCDF file from S3 directly using a file-like python object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:    (latitude: 7001, longitude: 10001, time: 1)\n",
       "Coordinates:\n",
       "  * time       (time) datetime64[ns] 2014-01-01\n",
       "  * latitude   (latitude) float64 10.0 9.99 9.98 9.97 ... -59.98 -59.99 -60.0\n",
       "  * longitude  (longitude) float64 80.0 80.01 80.02 80.03 ... 180.0 180.0 180.0\n",
       "Data variables:\n",
       "    K_490      (time, latitude, longitude) float32 ...\n",
       "Attributes:\n",
       "    history:      File initialised at 2015-12-16T15:53:48.785210\\nInitialised...\n",
       "    Conventions:  CF-1.6\n",
       "    source_path:  imos-srs/archive/oc/aqua/1d/v201508/2014/01/A20140101.L2OC_..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3path='imos-data/IMOS/SRS/OC/gridded/aqua/P1D/2014/01/A.P1D.20140101T000000Z.aust.K_490.nc'\n",
    "ds = xr.open_dataset(fs.open(s3path),engine='h5netcdf')\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import holoviews as hv\n",
    "import hvplot.xarray\n",
    "import hvplot.pandas\n",
    "import geoviews as gv\n",
    "import numpy as np\n",
    "import cartopy\n",
    "from cartopy import crs as ccrs\n",
    "\n",
    "hv.notebook_extension('bokeh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "plate=ccrs.PlateCarree(central_longitude=133.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%opts Overlay [width=700 height=400]\n",
    "%%opts Image [projection=plate]\n",
    "\n",
    "(ds.K_490.hvplot('longitude','latitude',groupby='time',dynamic=True, rasterize=True, cmap='magma',crs=ccrs.PlateCarree()).redim(K_490=dict(range=(0, 0.5))))*\\\n",
    "(gv.feature.coastline.geoms('10m')).opts(color='black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:c3dis2] *",
   "language": "python",
   "name": "conda-env-c3dis2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}