diff --git a/.gitignore b/.gitignore index 06e8e1e..8b65f47 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,17 @@ .Rhistory .RData .Ruserdata -Data/np_arrays/** Output/Pred/** *.pyc /.quarto/ + +Data/img/* +!Data/img/Gafanha_Low_LowRes.tif + +Data/shp/* +!Data/shp/mask/Gafanha_Low.shp +!Data/shp/mask/Gafanha_Low.cpg +!Data/shp/mask/Gafanha_Low.dbf +!Data/shp/mask/Gafanha_Low.prj +!Data/shp/mask/Gafanha_Low.qmd +!Data/shp/mask/Gafanha_Low.shx \ No newline at end of file diff --git a/Data/Training/DISCOV_Invasea_V3.csv b/Data/Training/DISCOV_Invasea_V3.csv new file mode 100644 index 0000000..0727bff --- /dev/null +++ b/Data/Training/DISCOV_Invasea_V3.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba28ca1d5b85dd41e74738bba29d218c4c8f605b8fff717abf01eb9bb6f267d4 +size 284123091 diff --git a/Data/shp/Gafanha_Low.cpg b/Data/shp/Gafanha_Low.cpg deleted file mode 100644 index 105bf61..0000000 --- a/Data/shp/Gafanha_Low.cpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3ad3031f5503a4404af825262ee8232cc04d4ea6683d42c5dd0a2f2a27ac9824 -size 5 diff --git a/Data/shp/Gafanha_Low.dbf b/Data/shp/Gafanha_Low.dbf deleted file mode 100644 index 5fd597b..0000000 --- a/Data/shp/Gafanha_Low.dbf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:92e338f30570d80ff7de12d97486b12b664cc30ff907bb01b42b7da636eb55ed -size 129 diff --git a/Data/shp/Gafanha_Low.prj b/Data/shp/Gafanha_Low.prj deleted file mode 100644 index 2098c4d..0000000 --- a/Data/shp/Gafanha_Low.prj +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a02a27b1d1982c8516d83398e85a3c8b1aef1713c13ef4d84d7bde17430c07c4 -size 145 diff --git a/Data/shp/Gafanha_Low.qmd b/Data/shp/Gafanha_Low.qmd deleted file mode 100644 index 02557d3..0000000 --- a/Data/shp/Gafanha_Low.qmd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6576a7fce39d3f5b7ac142034447d511cc389a2f51c8863beda0c9e88c0a3967 -size 667 diff --git a/Data/shp/Gafanha_Low.shp b/Data/shp/Gafanha_Low.shp deleted file mode 100644 index 7c8c685..0000000 --- a/Data/shp/Gafanha_Low.shp +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d40ccee8ca46729a3920d16b8fb87510b938b1d3995de83949022351533cc09f -size 303916 diff --git a/Data/shp/Gafanha_Low.shx b/Data/shp/Gafanha_Low.shx deleted file mode 100644 index df93c23..0000000 --- a/Data/shp/Gafanha_Low.shx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:926e399c704c0a62375bb428d33563949a29833a8c662035842e95f0fe6a8d62 -size 108 diff --git a/Models/DISCOV_BiCOME_reframed.pkl b/Models/DISCOV_BiCOME.pkl similarity index 100% rename from Models/DISCOV_BiCOME_reframed.pkl rename to Models/DISCOV_BiCOME.pkl diff --git a/Models/DISCOV_InvaSea.pkl b/Models/DISCOV_InvaSea.pkl new file mode 100644 index 0000000..709c9e1 Binary files /dev/null and b/Models/DISCOV_InvaSea.pkl differ diff --git a/Models/DISCOV_Invasea_V2.pkl b/Models/DISCOV_Invasea_V2.pkl new file mode 100644 index 0000000..80171f2 Binary files /dev/null and b/Models/DISCOV_Invasea_V2.pkl differ diff --git a/Scripts/Batch_prediction.ipynb b/Scripts/Batch_prediction.ipynb index dfc247d..8b591c9 100644 --- a/Scripts/Batch_prediction.ipynb +++ b/Scripts/Batch_prediction.ipynb @@ -32,7 +32,7 @@ " \"Lorois_50m_MicasenseV3\",\n", " \"NorthDike_high_multi\"] \n", "\n", - "models = [\"DISCOV_BiCOME_reframed\", \"DiSCOV_InvaSea_V3\"]\n", + "models = [\"DISCOV_BiCOME_reframed\", \"DiSCOV_InvaSea\"]\n", "\n", "\n", "batch_prediction(images, models)" diff --git a/Scripts/Model_Training.ipynb b/Scripts/Model_Training.ipynb index 1c03e50..211304f 100644 --- a/Scripts/Model_Training.ipynb +++ b/Scripts/Model_Training.ipynb @@ -2,23 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "tags": [] }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'fastbook'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m#hide\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfastbook\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mfastinference\u001b[39;00m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapi\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtypes\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m is_string_dtype, is_numeric_dtype, is_categorical_dtype\n", - "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'fastbook'" - ] - } - ], + "outputs": [], "source": [ "#hide\n", "from fastbook import *\n", @@ -73,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "metadata": { "tags": [] }, @@ -81,12 +69,12 @@ "source": [ "df_nn = pd.read_csv('../Data/Training/DISCOV_Invasea_V3.csv', low_memory=False,sep=\";\")\n", "\n", - "output_name = '../models/DISCOV_InvaSea_V3.pkl'" + "output_name = '../models/DISCOV_InvaSea.pkl'" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -95,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -298,7 +286,7 @@ "[1141506 rows x 23 columns]" ] }, - "execution_count": 11, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } diff --git a/Scripts/Prediction.ipynb b/Scripts/Prediction.ipynb index 2b028f4..0550cb8 100644 --- a/Scripts/Prediction.ipynb +++ b/Scripts/Prediction.ipynb @@ -64,19 +64,35 @@ } ], "source": [ - "filename = \"Gafanha_Low\"\n", + "filename = \"Gafanha_Low_LowRes\"\n", "output_name = \"Gaf_Low_BiCOME\"\n", "\n", - "learn = load_learner('../models/DISCOV_BiCOME_reframed.pkl')\n", + "learn = load_learner('../models/DISCOV_BiCOME.pkl')\n", "categories = learn.dls.vocab\n", "categories" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[-8.743678352905512,\n", + " 3.9002362204679466e-07,\n", + " 0.0,\n", + " 40.59793313496078,\n", + " 0.0,\n", + " -2.9730920060172317e-07]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "with fiona.open('../Data/shp/mask/' + filename + '.shp', \"r\") as shapefile:\n", " shapes = [feature[\"geometry\"] for feature in shapefile]\n", @@ -91,12 +107,13 @@ "\n", "lowtif = gdal.Open('../Data/img/' + filename + '.tif')\n", "proj = lowtif.GetProjection()\n", - "del lowtif \n" + "del lowtif \n", + "gt" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -128,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -148,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -158,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +184,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -178,17 +195,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(24779379, 22)" + "(2463151, 22)" ] }, + "execution_count": 11, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -197,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -206,7 +224,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -254,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -264,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -273,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -289,7 +307,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -298,8 +316,9 @@ "list" ] }, + "execution_count": 17, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -308,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -318,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -327,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -360,32 +379,32 @@ " \n", " 0\n", " 2\n", - " 70271\n", - " 0.479743\n", + " 10063\n", + " 0.996501\n", " \n", " \n", " 1\n", - " 4\n", - " 76621\n", - " 0.596987\n", + " 2\n", + " 10064\n", + " 0.993642\n", " \n", " \n", " 2\n", - " 4\n", - " 76622\n", - " 0.353299\n", + " 2\n", + " 11827\n", + " 0.999338\n", " \n", " \n", " 3\n", - " 4\n", - " 82971\n", - " 0.771637\n", + " 2\n", + " 11828\n", + " 0.998681\n", " \n", " \n", " 4\n", " 2\n", - " 82972\n", - " 0.623290\n", + " 11829\n", + " 0.996831\n", " \n", " \n", " ...\n", @@ -394,59 +413,60 @@ " ...\n", " \n", " \n", - " 24779374\n", - " 3\n", - " 53223154\n", - " 0.468945\n", + " 2463146\n", + " 2\n", + " 4625820\n", + " 0.998297\n", " \n", " \n", - " 24779375\n", + " 2463147\n", " 2\n", - " 53223155\n", - " 0.919087\n", + " 4625821\n", + " 0.997265\n", " \n", " \n", - " 24779376\n", - " 4\n", - " 53223156\n", - " 0.312370\n", + " 2463148\n", + " 2\n", + " 4627579\n", + " 0.995977\n", " \n", " \n", - " 24779377\n", - " 4\n", - " 53223157\n", - " 0.548968\n", + " 2463149\n", + " 2\n", + " 4627580\n", + " 0.998293\n", " \n", " \n", - " 24779378\n", + " 2463150\n", " 2\n", - " 53229504\n", - " 0.924912\n", + " 4627581\n", + " 0.998905\n", " \n", " \n", "\n", - "

24779379 rows × 3 columns

\n", + "

2463151 rows × 3 columns

\n", "" ], "text/plain": [ - " Pred_ID ID Prob\n", - "0 2 70271 0.479743\n", - "1 4 76621 0.596987\n", - "2 4 76622 0.353299\n", - "3 4 82971 0.771637\n", - "4 2 82972 0.623290\n", - "... ... ... ...\n", - "24779374 3 53223154 0.468945\n", - "24779375 2 53223155 0.919087\n", - "24779376 4 53223156 0.312370\n", - "24779377 4 53223157 0.548968\n", - "24779378 2 53229504 0.924912\n", + " Pred_ID ID Prob\n", + "0 2 10063 0.996501\n", + "1 2 10064 0.993642\n", + "2 2 11827 0.999338\n", + "3 2 11828 0.998681\n", + "4 2 11829 0.996831\n", + "... ... ... ...\n", + "2463146 2 4625820 0.998297\n", + "2463147 2 4625821 0.997265\n", + "2463148 2 4627579 0.995977\n", + "2463149 2 4627580 0.998293\n", + "2463150 2 4627581 0.998905\n", "\n", - "[24779379 rows x 3 columns]" + "[2463151 rows x 3 columns]" ] }, + "execution_count": 20, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -455,7 +475,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -464,7 +484,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -473,7 +493,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -582,102 +602,103 @@ " ...\n", " \n", " \n", - " 53232045\n", + " 4634885\n", " NaN\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", " NaN\n", - " 53232045\n", + " 4634885\n", " NaN\n", " NaN\n", " \n", " \n", - " 53232046\n", + " 4634886\n", " NaN\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", " NaN\n", - " 53232046\n", + " 4634886\n", " NaN\n", " NaN\n", " \n", " \n", - " 53232047\n", + " 4634887\n", " NaN\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", " NaN\n", - " 53232047\n", + " 4634887\n", " NaN\n", " NaN\n", " \n", " \n", - " 53232048\n", + " 4634888\n", " NaN\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", " NaN\n", - " 53232048\n", + " 4634888\n", " NaN\n", " NaN\n", " \n", " \n", - " 53232049\n", + " 4634889\n", " NaN\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", " NaN\n", - " 53232049\n", + " 4634889\n", " NaN\n", " NaN\n", " \n", " \n", "\n", - "

53232050 rows × 25 columns

\n", + "

4634890 rows × 25 columns

\n", "" ], "text/plain": [ - " Reflectance_444 Reflectance_475 Reflectance_531 Reflectance_560 \\\n", - "0 NaN NaN NaN NaN \n", - "1 NaN NaN NaN NaN \n", - "2 NaN NaN NaN NaN \n", - "3 NaN NaN NaN NaN \n", - "4 NaN NaN NaN NaN \n", - "... ... ... ... ... \n", - "53232045 NaN NaN NaN NaN \n", - "53232046 NaN NaN NaN NaN \n", - "53232047 NaN NaN NaN NaN \n", - "53232048 NaN NaN NaN NaN \n", - "53232049 NaN NaN NaN NaN \n", + " Reflectance_444 Reflectance_475 Reflectance_531 Reflectance_560 \\\n", + "0 NaN NaN NaN NaN \n", + "1 NaN NaN NaN NaN \n", + "2 NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN \n", + "4 NaN NaN NaN NaN \n", + "... ... ... ... ... \n", + "4634885 NaN NaN NaN NaN \n", + "4634886 NaN NaN NaN NaN \n", + "4634887 NaN NaN NaN NaN \n", + "4634888 NaN NaN NaN NaN \n", + "4634889 NaN NaN NaN NaN \n", "\n", - " ... NDVI_Stan ID Pred_ID Prob \n", - "0 ... NaN 0 NaN NaN \n", - "1 ... NaN 1 NaN NaN \n", - "2 ... NaN 2 NaN NaN \n", - "3 ... NaN 3 NaN NaN \n", - "4 ... NaN 4 NaN NaN \n", - "... ... ... ... ... ... \n", - "53232045 ... NaN 53232045 NaN NaN \n", - "53232046 ... NaN 53232046 NaN NaN \n", - "53232047 ... NaN 53232047 NaN NaN \n", - "53232048 ... NaN 53232048 NaN NaN \n", - "53232049 ... NaN 53232049 NaN NaN \n", + " ... NDVI_Stan ID Pred_ID Prob \n", + "0 ... NaN 0 NaN NaN \n", + "1 ... NaN 1 NaN NaN \n", + "2 ... NaN 2 NaN NaN \n", + "3 ... NaN 3 NaN NaN \n", + "4 ... NaN 4 NaN NaN \n", + "... ... ... ... ... ... \n", + "4634885 ... NaN 4634885 NaN NaN \n", + "4634886 ... NaN 4634886 NaN NaN \n", + "4634887 ... NaN 4634887 NaN NaN \n", + "4634888 ... NaN 4634888 NaN NaN \n", + "4634889 ... NaN 4634889 NaN NaN \n", "\n", - "[53232050 rows x 25 columns]" + "[4634890 rows x 25 columns]" ] }, + "execution_count": 23, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -686,7 +707,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -695,7 +716,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -704,17 +725,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(10, 8383, 6350)" + "(10, 2626, 1765)" ] }, + "execution_count": 26, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -723,7 +745,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -733,7 +755,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -749,11 +771,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Directory ../Output/Pred created.\n" + ] + } + ], "source": [ "# export\n", + "def ensure_directory_exists(directory_path):\n", + " if not os.path.exists(directory_path):\n", + " os.makedirs(directory_path)\n", + " print(f\"Directory {directory_path} created.\")\n", + " else:\n", + " print(f\"Directory {directory_path} already exists.\")\n", + "\n", + "ensure_directory_exists(\"../Output/Pred\") \n", + "\n", "driver = gdal.GetDriverByName(\"GTiff\")\n", "driver.Register()\n", "outds = driver.Create(\"../Output/Pred/\" + output_name +\"_prob.tif\", xsize = Prob_ras.shape[1],\n",