terra_mhub/papermill_notebooks/idc_seg_dicom_combination.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BA-Lmt-oAWjy"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "u6IZUFle7apI",
    "outputId": "83c8d7f8-7f47-4eda-f0c2-4fdc94498c30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "lz4 is already the newest version (1.9.3-2build2).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 56 not upgraded.\n"
     ]
    }
   ],
   "source": [
    "# !apt-get install lz4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1O9UNZct7e_9"
   },
   "outputs": [],
   "source": [
    "# %%capture\n",
    "# !pip install --upgrade duckdb papermill s5cmd dcm2niix pyplastimatch pyradiomics \\\n",
    "# highdicom pydicom seg-metrics idc-index itk SimpleITK pyaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qLB3dBkB7c6t"
   },
   "outputs": [],
   "source": [
    "# %%capture\n",
    "# from pyplastimatch.utils.install import install_precompiled_binaries\n",
    "# install_precompiled_binaries()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jYGs93hb7cQJ",
    "outputId": "93299fb4-eb15-474e-90df-7039819b611a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-09-05 18:55:23--  https://github.com/QIICR/dcmqi/releases/download/v1.3.4/dcmqi-1.3.4-linux.tar.gz\n",
      "Resolving github.com (github.com)... 140.82.113.4\n",
      "Connecting to github.com (github.com)|140.82.113.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/50675718/100afc23-a279-4a81-9202-a77bf1e574b9?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20240905%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240905T185523Z&X-Amz-Expires=300&X-Amz-Signature=a2994b53bd3cd6888f3c6ab051ab217f976a64ee907d4230d147e614a8270ad4&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=50675718&response-content-disposition=attachment%3B%20filename%3Ddcmqi-1.3.4-linux.tar.gz&response-content-type=application%2Foctet-stream [following]\n",
      "--2024-09-05 18:55:23--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/50675718/100afc23-a279-4a81-9202-a77bf1e574b9?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20240905%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240905T185523Z&X-Amz-Expires=300&X-Amz-Signature=a2994b53bd3cd6888f3c6ab051ab217f976a64ee907d4230d147e614a8270ad4&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=50675718&response-content-disposition=attachment%3B%20filename%3Ddcmqi-1.3.4-linux.tar.gz&response-content-type=application%2Foctet-stream\n",
      "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
      "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 40472151 (39M) [application/octet-stream]\n",
      "Saving to: ‘dcmqi-1.3.4-linux.tar.gz’\n",
      "\n",
      "\r\n",
      "dcmqi-1.3.4-linux.t   0%[                    ]       0  --.-KB/s               \r\n",
      "dcmqi-1.3.4-linux.t 100%[===================>]  38.60M  --.-KB/s    in 0.1s    \n",
      "\n",
      "2024-09-05 18:55:23 (366 MB/s) - ‘dcmqi-1.3.4-linux.tar.gz’ saved [40472151/40472151]\n",
      "\n",
      "dcmqi-1.3.4-linux/lib/\n",
      "dcmqi-1.3.4-linux/lib/dcmqi-1.3/\n",
      "dcmqi-1.3.4-linux/lib/dcmqi-1.3/libITKniftiio-5.3.a\n",
      "dcmqi-1.3.4-linux/lib/dcmqi-1.3/libITKznz-5.3.a\n",
      "dcmqi-1.3.4-linux/share/\n",
      "dcmqi-1.3.4-linux/share/doc/\n",
      "dcmqi-1.3.4-linux/share/doc/ITK-5.3/\n",
      "dcmqi-1.3.4-linux/share/doc/ITK-5.3/itksys/\n",
      "dcmqi-1.3.4-linux/share/doc/ITK-5.3/itksys/Copyright.txt\n",
      "dcmqi-1.3.4-linux/bin/\n",
      "dcmqi-1.3.4-linux/bin/itkimage2paramap\n",
      "dcmqi-1.3.4-linux/bin/tid1500writer.xml\n",
      "dcmqi-1.3.4-linux/bin/paramap2itkimage.xml\n",
      "dcmqi-1.3.4-linux/bin/tid1500reader\n",
      "dcmqi-1.3.4-linux/bin/itkimage2paramap.xml\n",
      "dcmqi-1.3.4-linux/bin/paramap2itkimage\n",
      "dcmqi-1.3.4-linux/bin/segimage2itkimage.xml\n",
      "dcmqi-1.3.4-linux/bin/segimage2itkimage\n",
      "dcmqi-1.3.4-linux/bin/itkimage2segimage.xml\n",
      "dcmqi-1.3.4-linux/bin/tid1500reader.xml\n",
      "dcmqi-1.3.4-linux/bin/itkimage2segimage\n",
      "dcmqi-1.3.4-linux/bin/tid1500writer\n"
     ]
    }
   ],
   "source": [
    "# dcmqi_release_url = \"https://github.com/QIICR/dcmqi/releases/download/v1.3.4/dcmqi-1.3.4-linux.tar.gz\"\n",
    "# dcmqi_download_path = \"dcmqi-1.3.4-linux.tar.gz\"\n",
    "# dcmqi_path = \"dcmqi-1.3.4-linux\"\n",
    "# !wget -O $dcmqi_download_path $dcmqi_release_url\\\n",
    "# && tar -xvf $dcmqi_download_path\\\n",
    "# && mv $dcmqi_path/bin/* /bin\\\n",
    "# && rm -r $dcmqi_download_path $dcmqi_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3Z6QNy1l7ZES"
   },
   "outputs": [],
   "source": [
    "# Imports\n",
    "\n",
    "import subprocess\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "import glob\n",
    "\n",
    "import duckdb\n",
    "import yaml\n",
    "import numpy as np\n",
    "\n",
    "import shutil\n",
    "from urllib.request import urlopen\n",
    "\n",
    "import pyplastimatch as pypla\n",
    "\n",
    "import math\n",
    "import pandas as pd\n",
    "import SimpleITK as sitk\n",
    "import seg_metrics.seg_metrics as sg\n",
    "\n",
    "import pydicom\n",
    "# Packages for the structured report\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import highdicom as hd\n",
    "\n",
    "from pydicom.uid import generate_uid\n",
    "from pydicom.filereader import dcmread\n",
    "from pydicom.sr.codedict import codes\n",
    "\n",
    "\n",
    "from idc_index import index\n",
    "client = index.IDCClient()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yJLRHiDh-7J_"
   },
   "source": [
    "# Global Variables\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "g98SFJpD1DoH",
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "#absolute path output\n",
    "OUTPUT_PATH = \"\"#\"OUTPUT_EVAL\"\n",
    "#Combination variables -- indicate whole prostate gland code\n",
    "## if whole prostate gland code not present, combine all the other segments to create whole  prostate gland segment\n",
    "dicomCodeValuesProstate_lst = []#[\"41216001\"]\n",
    "dicomCodeMeaningProstate_lst = []#[\"Prostatic structure\"]\n",
    "dicomCodingSchemeDesignatorProstate_lst = []#[\"SCT\"]\n",
    "idcSegSeriesInstanceUIDs = []#list([\"1.2.276.0.7230010.3.1.3.0.40454.1698534917.949757\"])\n",
    "res_scheme_format = \"\"#nii,nrrd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IF2GsyQS8B-N"
   },
   "outputs": [],
   "source": [
    "#input and output path definition\n",
    "base_folder_idc_combination_prostate = os.path.join(OUTPUT_PATH, \"seg_prostate_gen\")\n",
    "base_idc_output_nii_folder = os.path.join(base_folder_idc_combination_prostate, \"output_nii_combined\")\n",
    "base_idc_output_dicom_folder = os.path.join(base_folder_idc_combination_prostate,\"output_nii_combined_resampled_dicom/\")\n",
    "\n",
    "path_eval_csv = os.path.join(OUTPUT_PATH, \"output_eval.csv\")\n",
    "\n",
    "#IDC EVALUATION DATA\n",
    "## Image\n",
    "IDC_IMAGE = os.path.join(OUTPUT_PATH,\"idc_image_data/dicom\")\n",
    "IDC_IMAGE_NII = os.path.join(OUTPUT_PATH,\"idc_image_data/nii\")\n",
    "IDC_IMAGE_NRRD = os.path.join(OUTPUT_PATH,\"idc_image_data/nrrd\")\n",
    "IDC_IMAGE_RESAMPLED = os.path.join(OUTPUT_PATH,\"idc_image_data/nii_res\")\n",
    "# SEG\n",
    "IDC_SEG = os.path.join(OUTPUT_PATH,\"idc_seg_data/dicom\")\n",
    "IDC_SEG_NII = os.path.join(OUTPUT_PATH,\"idc_seg_data/nii\")\n",
    "IDC_SEG_RESAMPLED = os.path.join(OUTPUT_PATH,\"idc_seg_data/nii_res\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DKiZDSKBIT0Q"
   },
   "outputs": [],
   "source": [
    "def setup_folders():\n",
    "\t\"\"\"\n",
    "\tcreates environment data and results folder, first deletes current ones and then creates new ones\n",
    "\t\"\"\"\n",
    "\tfor folder_setup in [OUTPUT_PATH, IDC_IMAGE, IDC_IMAGE_NII, IDC_IMAGE_NRRD, \n",
    "                         base_idc_output_dicom_folder, base_idc_output_nii_folder,\n",
    "                         IDC_IMAGE_RESAMPLED, IDC_SEG, IDC_SEG_NII, IDC_SEG_RESAMPLED]:\n",
    "\t\t# !rm -rf $folder_setup\n",
    "\t\t!mkdir -p $folder_setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ETni0gcGBxNu"
   },
   "outputs": [],
   "source": [
    "setup_folders()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BwqI2QIHoJiq"
   },
   "source": [
    "# Utility functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CeyD64ZnyeAA"
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KDV_5w-1hYzw"
   },
   "outputs": [],
   "source": [
    "def compute_dice(pred_path, gt_path):\n",
    " #computation of dice score\n",
    "  out_plast = !plastimatch dice --dice $gt_path $pred_path\n",
    "  dice_score = [el[1] for el in out_plast.fields() if \"DICE\" in el[0]][0]#formatting by plastimatch dice output, retrieve of DSC\n",
    "  return float(dice_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OVyNQP9Cb2tu"
   },
   "outputs": [],
   "source": [
    "def compute_hsdff_95(pred_path, gt_path, labelID=1):\n",
    "\tout_plast = !plastimatch dice --hausdorff $gt_path $pred_path\n",
    "\thsdff = [el[5] for el in out_plast.fields() if \"Percent\" in el[0] and \"distance\" in el[3]][0]#formatting by plastimatch hsdff output, retrieve of hsdff\n",
    "\treturn float(hsdff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OSy98b7zmxaB"
   },
   "outputs": [],
   "source": [
    "def compute_hsdff_regular(pred_path, gt_path):\n",
    "\t#computation of dice score\n",
    "\tout_plast = !plastimatch dice --hausdorff $gt_path $pred_path\n",
    "\thsdff = [el[3] for el in out_plast.fields() if \"Hausdorff\" in el[0] and \"distance\" in el[1]][0]#formatting by plastimatch hsdff output, retrieve of hsdff\n",
    "\treturn float(hsdff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NtmROiLxPb7i"
   },
   "outputs": [],
   "source": [
    "#https://github.com/Jingnan-Jia/segmentation_metrics\n",
    "def compute_avg_surface_dist(pred_path, gt_path):\n",
    "\t# for only one metrics\n",
    "\timage_gt = sitk.ReadImage(gt_path)\n",
    "\timage_pred = sitk.ReadImage(pred_path)\n",
    "\tgt_spacing = image_gt.GetSpacing()\n",
    "\tgt_size = image_gt.GetSize()\n",
    "\tnew_size = [int(round(osz*ospc/nspc)) for osz,ospc,nspc in zip(gt_size, gt_spacing, image_pred.GetSpacing())]\n",
    "\tgt_image_new  = sitk.Resample(image_gt, new_size, sitk.Transform(), sitk.sitkNearestNeighbor,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\timage_gt.GetOrigin(), image_pred.GetSpacing(), image_gt.GetDirection(), 0,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\timage_gt.GetPixelID())\n",
    "\tmetrics = sg.write_metrics(labels=[1],  # exclude background if needed -- here only 1 label, prostate\n",
    "\t\t\t\t\t\t\t\t\t\tgdth_img=gt_image_new,\n",
    "\t\t\t\t\t\t\t\t\t\tpred_img=sitk.ReadImage(pred_path),\n",
    "\t\t\t\t\t\t\t\t\t\tcsv_file=None,\n",
    "\t\t\t\t\t\t\t\t\t\tspacing=list(sitk.ReadImage(pred_path).GetSpacing()),\n",
    "\t\t\t\t\t\t\t\t\t\tmetrics='msd')\n",
    "\treturn metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qI26kJzlr71b"
   },
   "outputs": [],
   "source": [
    "def compute_true_positive(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"TP\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KQRuRNDdsEyj"
   },
   "outputs": [],
   "source": [
    "def compute_true_negative(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"TN\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "owAxQV5OsICX"
   },
   "outputs": [],
   "source": [
    "def compute_false_negative(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"FN\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4R903vsbjPvr"
   },
   "outputs": [],
   "source": [
    "def compute_false_positive(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"FP\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YdZOyWpHjTdf"
   },
   "outputs": [],
   "source": [
    "def compute_sensitivity(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"SE\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "c26EZaR3sLk-"
   },
   "outputs": [],
   "source": [
    "def compute_specificity(pred_path, gt_path):\n",
    "\tout_plast = !plastimatch dice --all $gt_path $pred_path\n",
    "\treturn [el[1] for el in out_plast.fields() if \"SP\" in el[0]][0]#formatting by plastimatch hsdff output, retrieve of hsdff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EtveZiW5oMRY"
   },
   "source": [
    "## Conversion DICOM <---> NII"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2ig8O6X2oXRc"
   },
   "outputs": [],
   "source": [
    "def convert_image_dcm_to_nii(input_path, output_path_root, prefix=\"\", format=\"nii\"):\n",
    "\t\"\"\"\n",
    "\tConversion of DICOM MR data to NIFTI using dcm2niix\n",
    "\tinput_path : str, folder containing DICOM instances .dcm\n",
    "\toutput_path_root : str, output folder\n",
    "\tprefix : str, prefix for output file name\n",
    "\t\"\"\"\n",
    "\tif not os.path.exists(output_path_root):\n",
    "\t\t!mkdir -p $output_path_root\n",
    "\t!dcm2niix -z y -m y -f %i_{prefix} -o $output_path_root $input_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "px_GC146oRLV"
   },
   "source": [
    "## Resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resample_seg_with_sitk(input_seg_nifti_path, input_dicom_path, output_seg_path):\n",
    "  print(\"Reading Dicom directory:\", input_dicom_path)\n",
    "  reader = sitk.ImageSeriesReader()\n",
    "  dicom_names = reader.GetGDCMSeriesFileNames(input_dicom_path)\n",
    "  reader.SetFileNames(dicom_names)\n",
    "  target_dicom_img_itk = reader.Execute()\n",
    "  # define a sitk identity transform\n",
    "  identity_tfx = sitk.Transform(3, sitk.sitkIdentity)\n",
    "  seg_itk = sitk.ReadImage(input_seg_nifti_path)\n",
    "  # align the segmentation mask to the target dicom image (to get a uniform z height as per the target dicom image)\n",
    "  seg_aligned_itk = sitk.Resample(seg_itk, target_dicom_img_itk, identity_tfx, sitk.sitkNearestNeighbor, 0)\n",
    "  # overwrite output file with aligned image data\n",
    "  sitk.WriteImage(seg_aligned_itk, output_seg_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UnidOuJ9sxJl"
   },
   "outputs": [],
   "source": [
    "def resample_preds(input_path_nnunet_preds=\"\", input_path_t2_idc=\"\", output_path=\"\"):\n",
    "\t\"\"\"\n",
    "\tResampling of nnunet_preds to reference MR T2 volumes used as input image\n",
    "\tinput_path_nnunet_preds : str, folder containing nnunet preds\n",
    "\tinput_path_t2_idc : str, folder containing NIFTI IDC MR T2 volumes\n",
    "\toutput_path : str, folder output\n",
    "\t\"\"\"\n",
    "\tfor pred_path in sorted(glob.glob(os.path.join(input_path_nnunet_preds, \"*.nii.gz\"))):\n",
    "\t\tprint(f\"pred path : {pred_path}\")\n",
    "\t\tresample_args_to_t2_origin = {\"input\" : pred_path,#change to pred path if no largest_component_retrieval necessary\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\"output\" : os.path.join(output_path,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tf\"{pred_path.split('/')[-1][:-7]}.nii.gz\"),\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\"fixed\" : input_path_t2_idc,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\"interpolation\" : \"nn\"}\n",
    "\n",
    "\t\tpypla.resample(verbose = False, **resample_args_to_t2_origin)\n",
    "\t\tprint()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R9XGazYp8QKq"
   },
   "source": [
    "## Combine segs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DQ00_GHx8Rye"
   },
   "outputs": [],
   "source": [
    "def add_segments_json(input_json, segmentAttributesProstate_lst):\n",
    "\tassert os.path.exists(input_json)\n",
    "\tinp_json = json.load(open(input_json)).copy()\n",
    "\tprint(inp_json[\"segmentAttributes\"])\n",
    "\tprint(type(inp_json[\"segmentAttributes\"]))\n",
    "\ttemp_lst = inp_json[\"segmentAttributes\"].copy()\n",
    "\ttemp_lst.append(segmentAttributesProstate_lst)\n",
    "\tinp_json[\"segmentAttributes\"] = temp_lst\n",
    "\tprint(inp_json[\"segmentAttributes\"])\n",
    "\n",
    "\treturn inp_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YJiQmS3thjBh"
   },
   "outputs": [],
   "source": [
    "def retrieve_meta_df_from_json(meta_json_path):\n",
    "\tout_segment_list = []\n",
    "\tjson_dic = json.load(open(meta_json_path))\n",
    "\tfor segment_iter in json_dic[\"segmentAttributes\"]:\n",
    "\t\ttemp_dic = {}\n",
    "\t\ttemp_dic = segment_iter\n",
    "\t\tout_segment_list.append(temp_dic[0])\n",
    "\tout_df = pd.DataFrame.from_dict(out_segment_list)\n",
    "\tdf_SegmentedPropertyTypeCodeSequence = pd.json_normalize(out_df['SegmentedPropertyTypeCodeSequence']).rename(columns={'CodeMeaning': 'FindingSite_CodeMeaning',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'CodeValue':'FindingSite_CodeValue',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'CodingSchemeDesignator': 'FindingSite_CodingSchemeDesignator'})\n",
    "\tdf_SegmentedPropertyCategoryCodeSequence = pd.json_normalize(out_df['SegmentedPropertyCategoryCodeSequence']).rename(columns={'CodeMeaning': 'Finding_CodeMeaning',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'CodeValue':'Finding_CodeValue',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'CodingSchemeDesignator': 'Finding_CodingSchemeDesignator'})\n",
    "\tdf_columns_lst = list(out_df.columns.values)\n",
    "\tcols_to_retrieve = []\n",
    "\tfor col_test in [\"SegmentAlgorithmName\",\"SegmentAlgorithmType\",\n",
    "\t\t\t\t\t\t\t\t\t \"SegmentDescription\",\"SegmentLabel\"]:\n",
    "\t\tif col_test in df_columns_lst:\n",
    "\t\t\t\tcols_to_retrieve.append(col_test)\n",
    "\t\telse:\n",
    "\t\t\tpass\n",
    "\tdf_final = df_SegmentedPropertyTypeCodeSequence.join(df_SegmentedPropertyCategoryCodeSequence).join(out_df[[\"labelID\"]\\\n",
    "\t\t+ cols_to_retrieve])\n",
    "\tdf_final[\"segment\"] = df_final[\"FindingSite_CodeMeaning\"]\n",
    "\tdf_final[\"SeriesDescription\"] = json_dic[\"SeriesDescription\"]\n",
    "\tdf_final[\"SeriesNumber\"] = json_dic[\"SeriesNumber\"]\n",
    "\tdf_final[\"BodyPartExamined\"] = json_dic[\"BodyPartExamined\"]\n",
    "\tdf_final = df_final.reset_index().drop(columns=[\"index\"])\n",
    "\treturn df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Rr5ELJhV_qtn"
   },
   "outputs": [],
   "source": [
    "def generateProstateDcmqiMetadata(labelID, SegmentAlgorithmType,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentAlgorithmName,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentLabel,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentDescription,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyCategoryCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyCategoryCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyCategoryCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyTypeCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyTypeCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tSegmentedPropertyTypeCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\trecommendedDisplayRGBValue):\n",
    "\tout_lst = []\n",
    "\tif SegmentAlgorithmName is not None:\n",
    "\t\tout_lst.append({\"SegmentAlgorithmName\" : SegmentAlgorithmName,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentAlgorithmType\" : SegmentAlgorithmType,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentLabel\" : SegmentLabel,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentDescription\" : SegmentDescription,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentedPropertyCategoryCodeSequence\" : {\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodeMeaning\" : SegmentedPropertyCategoryCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodeValue\" : SegmentedPropertyCategoryCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodingSchemeDesignator\" : SegmentedPropertyCategoryCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t},\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentedPropertyTypeCodeSequence\" : {\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodeMeaning\" : SegmentedPropertyTypeCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodeValue\" : SegmentedPropertyTypeCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodingSchemeDesignator\" : SegmentedPropertyTypeCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t},\n",
    "\t\t\t\t\t\t\t\t\t\t\"labelID\" : int(labelID),\n",
    "\t\t\t\t\t\t\t\t\t\t\"recommendedDisplayRGBValue\" : [int(x) for x in recommendedDisplayRGBValue]})\n",
    "\telse:\n",
    "\t\tout_lst.append({\"SegmentAlgorithmType\" : SegmentAlgorithmType,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentLabel\" : SegmentLabel,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentDescription\" : SegmentDescription,\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentedPropertyCategoryCodeSequence\" : {\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodeMeaning\" : SegmentedPropertyCategoryCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodeValue\" : SegmentedPropertyCategoryCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\"CodingSchemeDesignator\" : SegmentedPropertyCategoryCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t},\n",
    "\t\t\t\t\t\t\t\t\t\t\"SegmentedPropertyTypeCodeSequence\" : {\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodeMeaning\" : SegmentedPropertyTypeCodeSequenceCodeMeaning,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodeValue\" : SegmentedPropertyTypeCodeSequenceCodeValue,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\"CodingSchemeDesignator\" : SegmentedPropertyTypeCodeSequenceCodingSchemeDesignator,\n",
    "\t\t\t\t\t\t\t\t\t\t},\n",
    "\t\t\t\t\t\t\t\t\t\t\"labelID\" : int(labelID),\n",
    "\t\t\t\t\t\t\t\t\t\t\"recommendedDisplayRGBValue\" : [int(x) for x in recommendedDisplayRGBValue]})\n",
    "\treturn out_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YrLX0e5Zfee0"
   },
   "outputs": [],
   "source": [
    "def combine_ind_seg(input_nii_folder, output_file_path):\n",
    "\tassert os.path.exists(input_nii_folder)\n",
    "\tout_arr = np.zeros_like(sitk.GetArrayFromImage(sitk.ReadImage(glob.glob(os.path.join(input_nii_folder, \"*.nii.gz\"))[0])))\n",
    "\tfor segment_processed in sorted(glob.glob(os.path.join(input_nii_folder, \"*.nii.gz\"))):\n",
    "\t\tout_arr += sitk.GetArrayFromImage(sitk.ReadImage(segment_processed))\n",
    "\tres_img = sitk.GetImageFromArray(out_arr)\n",
    "\tres_img.CopyInformation(sitk.ReadImage(glob.glob(os.path.join(input_nii_folder, \"*.nii.gz\"))[0]))\n",
    "\t# Save output segment\n",
    "\tsitk.WriteImage(res_img, output_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jpldrJRlF6C3"
   },
   "outputs": [],
   "source": [
    "def combine_regions_to_prostate(input_dcmqi_nii_folder,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tinput_dcmqi_dicom_seg_path,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tinput_ref_image_nii_file,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\toutput_nii_folder,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\toutput_dicom_folder,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tdicom_codeValuesProstate_lst,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tdicom_CodingSchemeDesignatorProstate_lst):\n",
    "\tfor path_out_folder in [output_dicom_folder, output_nii_folder]:\n",
    "\t\tif not os.path.exists(path_out_folder):\n",
    "\t\t\t!mkdir -p $path_out_folder\n",
    "\t\telse:\n",
    "\t\t\t!rm -rf $path_out_folder\n",
    "\t\t\t!mkdir -p $path_out_folder\n",
    "\tinput_segments_path_lst = glob.glob(os.path.join(input_dcmqi_nii_folder, \"**\", \"*.nii.gz\"), recursive=True)\n",
    "\tinput_segments_path_folder = \"/\".join(input_segments_path_lst[0].split(\"/\")[:-1])\n",
    "\tprint(f\"INPUT SEGS LIST : {input_segments_path_lst}\")\n",
    "\tprint(f\"input_segments_path_folder : {input_segments_path_folder}\")\n",
    "\tout_seg_image = np.zeros_like(sitk.GetArrayFromImage(sitk.ReadImage(input_segments_path_lst[0])))\n",
    "\tinput_metadata_file = glob.glob(os.path.join(input_dcmqi_nii_folder, \"**\", \"*.json\"), recursive=True)[0]\n",
    "\tdcmqi_json_df = retrieve_meta_df_from_json(input_metadata_file)\n",
    "\toutput_seg_dcm_filename = input_dcmqi_dicom_seg_path.split(\"/\")[-1]\n",
    "\tflag_ProstateGland = False\n",
    "\tsegCodeValuesLst = dcmqi_json_df.FindingSite_CodeValue.values\n",
    "\tsegCodingSchemeDesignatorLst = dcmqi_json_df.FindingSite_CodingSchemeDesignator.values\n",
    "\tfor analyzed_segmentCodeValue, analyzed_segmentCodingSchemeDesignator in zip(segCodeValuesLst, segCodingSchemeDesignatorLst):\n",
    "\t\tif analyzed_segmentCodeValue in dicom_codeValuesProstate_lst and analyzed_segmentCodingSchemeDesignator in dicom_CodingSchemeDesignatorProstate_lst:\n",
    "\t\t\tflag_ProstateGland = True\n",
    "\t\telse:\n",
    "\t\t\tpass\n",
    "\tprint(f\"Flag ProstateGland : {flag_ProstateGland}\")\n",
    "\tif (flag_ProstateGland):\n",
    "\t\tprint(\"whole prostate gland segment already present!\")\n",
    "\t\t# copy dicom seg data\n",
    "\t\tshutil.copy(input_dcmqi_dicom_seg_path, os.path.join(output_dicom_folder, output_seg_dcm_filename))\n",
    "\t\t#copy nii data\n",
    "\t\tfor copy_seg_nii in glob.glob(os.path.join(input_dcmqi_nii_folder, \"**\", \"*.nii.gz\"), recursive=True):\n",
    "\t\t\tprint(f\"segment being copied : {copy_seg_nii}\")\n",
    "\t\t\tout_path_seg_nii = os.path.join(output_nii_folder, copy_seg_nii.split(\"/\")[-1])\n",
    "\t\t\tshutil.copy(copy_seg_nii, out_path_seg_nii)\n",
    "\t\tshutil.copy(os.path.join(input_dcmqi_nii_folder, \"meta.json\"),\n",
    "\t\t\t\t\t\t\t\tos.path.join(output_nii_folder, \"meta.json\"))\n",
    "\telse:\n",
    "\t\tfor segCodeValue, segCodingSchemeDesignator in zip(segCodeValuesLst, segCodingSchemeDesignatorLst):\n",
    "\t\t\tsegs_path_lst = glob.glob(os.path.join(input_dcmqi_nii_folder,'**', '*.nii.gz'), recursive=True)\n",
    "\t\t\tfilter_segLabelID = dcmqi_json_df[(dcmqi_json_df.FindingSite_CodeValue == segCodeValue)\\\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t&(dcmqi_json_df.FindingSite_CodingSchemeDesignator == segCodingSchemeDesignator)].labelID.values[0]\n",
    "\t\t\tfilterLabelID_path = glob.glob(os.path.join(input_segments_path_folder, f\"{filter_segLabelID}.nii.gz\"))[0]\n",
    "\t\t\ttemp_arr = sitk.GetArrayFromImage(sitk.ReadImage(filterLabelID_path)).copy()\n",
    "\t\t\tout_seg_image += temp_arr\n",
    "\t\t\toutLabelIDpath = os.path.join(output_nii_folder, filterLabelID_path.split(\"/\")[-1])\n",
    "\t\t\t# !cp $filterLabelID_path $outLabelIDpath\n",
    "\t\t\tprint(f\"filter_segLabelID : {filter_segLabelID}\")\n",
    "\t\t\tprint(f\"filterLabelID_path : {filterLabelID_path}\")\n",
    "\t\t\tprint(f\"outLabelIDpath : {outLabelIDpath}\")\n",
    "\t\t\tshutil.copy(filterLabelID_path, outLabelIDpath)\n",
    "\t\tnewLabelID = np.sort(dcmqi_json_df.labelID.values, axis=0)[-1] + 1\n",
    "\t\tout_seg_image[out_seg_image !=0] = newLabelID\n",
    "\t\tres_img = sitk.GetImageFromArray(out_seg_image)\n",
    "\t\tres_img.CopyInformation(sitk.ReadImage(input_segments_path_lst[0]))\n",
    "\t\t# Save output segment\n",
    "\t\tsitk.WriteImage(res_img, os.path.join(output_nii_folder, f\"{newLabelID}.nii.gz\"))\n",
    "\t\t#generate new dcmqi json\n",
    "\t\tif str(dcmqi_json_df.SegmentAlgorithmType.unique()[0]) == \"MANUAL\":\n",
    "\t\t\tSegmentAlgorithmName = None\n",
    "\t\telse:\n",
    "\t\t\tSegmentAlgorithmName = str(dcmqi_json_df.SegmentAlgorithmName.unique()[0])\n",
    "\t\tsegmentAttributesProstate_lst  = generateProstateDcmqiMetadata(labelID = int(newLabelID),\n",
    "\t\tSegmentAlgorithmType = str(dcmqi_json_df.SegmentAlgorithmType.unique()[0]),#\"AUTOMATIC\",\n",
    "\t\tSegmentAlgorithmName = SegmentAlgorithmName,#\"Monai-prostate158\",\n",
    "\t\tSegmentLabel=\"Prostate\",\n",
    "\t\tSegmentDescription=\"Prostate\",\n",
    "\t\tSegmentedPropertyCategoryCodeSequenceCodeValue = \"123037004\",\n",
    "\t\tSegmentedPropertyCategoryCodeSequenceCodingSchemeDesignator = \"SCT\",\n",
    "\t\tSegmentedPropertyCategoryCodeSequenceCodeMeaning = \"Anatomical Structure\",\n",
    "\t\tSegmentedPropertyTypeCodeSequenceCodeValue = \"41216001\",\n",
    "\t\tSegmentedPropertyTypeCodeSequenceCodingSchemeDesignator = \"SCT\",\n",
    "\t\tSegmentedPropertyTypeCodeSequenceCodeMeaning = \"Prostate\",\n",
    "\t\trecommendedDisplayRGBValue = [128,0,128])#purple\n",
    "\t\tcombined_json = add_segments_json(glob.glob(os.path.join(input_dcmqi_nii_folder,\n",
    "\t\t\t\t\"**\", \"*.json\"), recursive=True)[0], segmentAttributesProstate_lst)\n",
    "\t\twith open(os.path.join(output_nii_folder, \"meta.json\"), 'w', encoding='utf-8') as f:\n",
    "\t\t\tjson.dump(combined_json, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aLWNC-Pw9EW-"
   },
   "source": [
    "# Combine IDC SEGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sEqxutrY8xGl",
    "outputId": "5bd7c0a6-522b-4dd3-8d1f-052578312bd2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading data:   0%|          | 0.00/540k [00:00<?, ?B/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "serieUID image referenced : 1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dcmqi repository URL: https://github.com/QIICR/dcmqi revision: d88b025 tag: v1.3.4\n",
      "Loading DICOM SEG file OUTPUT_EVAL/idc_seg_data/dicom/1.2.276.0.7230010.3.1.3.0.40454.1698534917.949757/prostate_mri_us_biopsy/Prostate-MRI-US-Biopsy-1009/1.3.6.1.4.1.14519.5.2.1.208519745711386985148515509381757813785/SEG_1.2.276.0.7230010.3.1.3.0.40454.1698534917.949757/3f23933e-9f1a-4341-af96-199160be823d.dcm\n",
      "Row direction: 1 0 0\n",
      "Col direction: 0 1 0\n",
      "Z direction: 0 0 1\n",
      "Total frames: 60\n",
      "Total frames with unique IPP: 60\n",
      "Total overlapping frames: 0\n",
      "Origin: [-93.036, -53.588, 2.32527]\n",
      "Slice extent: 88.5\n",
      "Slice spacing: 1.5\n",
      "Will not merge segments: Splitting segments into 1 groups\n",
      "Writing itk image to OUTPUT_EVAL/idc_seg_data/nii/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/1.nii.gz ... done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading data: 100%|█████████▉| 8.08M/8.08M [00:00<00:00, 16.0MB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chris Rorden's dcm2niiX version v1.0.20220505  GCC11.4.0 x86-64 (64-bit Linux)\n",
      "Found 60 DICOM file(s)\n",
      "Convert 60 DICOM as OUTPUT_EVAL/idc_image_data/nii/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/Prostate-MRI-US-Biopsy-1009_1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671 (256x256x60x1)\n",
      "Conversion required 0.704190 seconds (0.704114 for core code).\n",
      "OUTPUT_EVAL/idc_seg_data/nii/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/1.nii.gz\n",
      "Flag ProstateGland : True\n",
      "whole prostate gland segment already present!\n",
      "segment being copied : OUTPUT_EVAL/idc_seg_data/nii_res/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/1.nii.gz\n",
      "dcmqi repository URL: https://github.com/QIICR/dcmqi revision: d88b025 tag: v1.3.4\n",
      "Loaded segmentation from OUTPUT_EVAL/idc_prostate_gen/output_nii_combined/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/1.nii.gz\n",
      "Searching recursively OUTPUT_EVAL/idc_image_data/dicom/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671 for DICOM files\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.323222986025842506659825571317352540898 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.168540175660454206854361946270411501649 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.324481168712737253968494712340749833418 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.265662737927103335368973701055136297013 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.23093004139554379403222038240262085387 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.146432160557729103405905197510065760724 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.155199631191170869002833642921971820782 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.17331828917810856993531745496722653404 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.31008063066212058321377080344492635764 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.91471212204721102978014015981090060996 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.90408067859016989770338646068748875033 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.238709608798605570269286101853795177830 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.114417697728567098864499249998408297973 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.304338496428987136873107791387808996034 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.78965188035634104402038076358393369210 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.57118268584432424138544802729798325142 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.168471303871429480131761918299519203553 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.171368784324369915351118108173500127384 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.146283501319279087472963710934239840809 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.26529419342537327564932146382808052788 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.274913502748061024058753027836664332354 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.37260171169082344179873586561793340862 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.203427398869061103637761191340274252549 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.219680555103495716799093792881905507479 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.176365502344988767466574184865716907501 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.18014304390146706765993797403162566157 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.118255445886287337986328865388495338206 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.318352381764344115806648377441337318141 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.317223520245071359463143188789062661060 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.301932448413827314812465467141539701670 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.54740790589547864718331571975333321275 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.261695037660126769091879125241215511759 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.135646956843848842562383419960376701956 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.318718460656268558019588747052428600351 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.13709902470563150529779452215494163798 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.163405417346567868836845504925777037606 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.36641028337153644497829528278374442654 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.78330697462922112185585440412693211451 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.283704541037334238090586504203887859402 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.244482972856430265891831738601348461134 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.220393903538926469893597724126058409752 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.89984744007793905968355400599943047382 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.247028681660257133918909493584566729568 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.233938321769048136100655751944049506023 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.37470058475874694050164748310725430635 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.56785710500156971662155585477985965630 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.97633596336409105062235228238725981120 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.237185511392926570593157534777891403928 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.73705995993426388000897158423197860338 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.172285208190418908501954192504338905565 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.116084554085719504977977864359492411211 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.67495853062853018474209056144051967315 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.150970366261133016145175303388779090015 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.338679028968306579823122626450914200443 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.20377560255581932065038185346534731786 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.232343008710793680937798631105771779196 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.251733119248899392291181755055110528035 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.271471694528644484553039402332321702174 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.222835598852131866353617727549177670732 mapped\n",
      "SOPInstanceUID 1.3.6.1.4.1.14519.5.2.1.276676632489628263233806880226676449737 mapped\n",
      "60 of 60 slices mapped to source DICOM images\n",
      "Found 2 label(s)\n",
      "Processing label 1\n",
      "Total non-empty slices that will be encoded in SEG for label 1 is 29\n",
      " (inclusive from 20 to 48)\n",
      "Populating segment label to Prostate\n",
      "Writing DICOM segmentation dataset\n",
      "Patching in extra meta information into DICOM dataset\n",
      "Saved segmentation as OUTPUT_EVAL/idc_prostate_gen/output_nii_combined_resampled_dicom/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/3f23933e-9f1a-4341-af96-199160be823d.dcm\n"
     ]
    }
   ],
   "source": [
    "for idc_expert_serieUID in idcSegSeriesInstanceUIDs:\n",
    "\ttry:\n",
    "\t\t#download expert SEG idc data\n",
    "\t\tout_image_dicom = os.path.join(IDC_SEG, idc_expert_serieUID)\n",
    "\t\t!mkdir -p $out_image_dicom\n",
    "\t\tclient.download_from_selection(seriesInstanceUID=list([idc_expert_serieUID]),\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tdownloadDir=out_image_dicom,)\n",
    "\t\tidc_seg_dicom_path = glob.glob(os.path.join(out_image_dicom, \"**\", \"*.dcm\"), recursive=True)[0]\n",
    "\t\tserieUID_image = str(pydicom.dcmread(idc_seg_dicom_path).ReferencedSeriesSequence[0].SeriesInstanceUID)\n",
    "\t\tprint(f\"serieUID image referenced : {serieUID_image}\")\n",
    "\t\tidc_output_nii_folder = os.path.join(IDC_SEG_NII, serieUID_image)\n",
    "\t\t!mkdir -p $idc_output_nii_folder\n",
    "\t\tassert os.path.exists(idc_seg_dicom_path)\n",
    "\t\tassert os.path.exists(idc_output_nii_folder)\n",
    "\t\t!segimage2itkimage --inputDICOM $idc_seg_dicom_path \\\n",
    "\t\t--outputDirectory $idc_output_nii_folder --outputType nii\n",
    "\t\t#download IDC reference image for downstream resampling\n",
    "\t\tout_image_dicom = os.path.join(IDC_IMAGE, serieUID_image)\n",
    "\t\t!rm -rf $out_image_dicom\n",
    "\t\t!mkdir -p $out_image_dicom\n",
    "\t\tclient.download_from_selection(seriesInstanceUID=list([serieUID_image]),\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tdownloadDir=out_image_dicom,)\n",
    "\t\t##\n",
    "\t\t#convert idc reference image to NIFTI/NRRD\n",
    "\t\tout_image_nii = os.path.join(IDC_IMAGE_NII, serieUID_image)\n",
    "\t\tout_image_nrrd = os.path.join(IDC_IMAGE_NRRD, serieUID_image)\n",
    "\t\t##nii\n",
    "\t\t!rm -rf $out_image_nii\n",
    "\t\t!mkdir -p $out_image_nii\n",
    "\t\t##nrrd\n",
    "\t\t!rm -rf $out_image_nrrd\n",
    "\t\t!mkdir -p $out_image_nrrd\n",
    "\t\t#convert image dicom to nii\n",
    "\t\t!dcm2niix -z y -m y -f %i_{serieUID_image} -o $out_image_nii $out_image_dicom\n",
    "\t\t#convert image to nrrd\n",
    "\t\tout_image_dicom_for_nrrd = \"/\".join(glob.glob(os.path.join(out_image_dicom, \"**\", \"*.dcm\"), recursive=True)[0].split(\"/\")[:-1])\n",
    "\t\tout_img_nrrd_file_path = os.path.join(out_image_nrrd, f\"{serieUID_image}.nrrd\")\n",
    "\t\tconvert_nrrd_args = {\"input\":out_image_dicom_for_nrrd,\n",
    "\t                     \"output-img\":out_img_nrrd_file_path}\n",
    "\t\tpypla.convert(verbose=True, **convert_nrrd_args)\n",
    "\t\tref_image_nii_path = glob.glob(os.path.join(out_image_nii, '*.nii.gz'))[0]\n",
    "\t\tref_image_nrrd_path = glob.glob(os.path.join(out_image_nrrd, '*.nrrd'))[0]\n",
    "\t\t#resample AI NIFTI SEG object to the reference image space\n",
    "\t\toutput_serie_resampled = os.path.join(IDC_SEG_RESAMPLED, serieUID_image)\n",
    "\t\t!rm -rf $output_serie_resampled\n",
    "\t\t!mkdir -p $output_serie_resampled\n",
    "\t\t#define resampling scheme\n",
    "\t\tif res_scheme_format == \"nrrd\":\n",
    "\t\t\tref_image_path = ref_image_nrrd_path\n",
    "\t\telif res_scheme_format == \"nii\":\n",
    "\t\t\tref_image_path = ref_image_nii_path\n",
    "\t\telse:\n",
    "\t\t\tref_image_path = ref_image_nii_path\n",
    "\t\tfor ai_segment_path in glob.glob(os.path.join(idc_output_nii_folder, \"*.nii.gz\")):\n",
    "\t\t\tprint(ai_segment_path)\n",
    "\t\t\tout_segment_path = os.path.join(output_serie_resampled, ai_segment_path.split('/')[-1])\n",
    "\t\t\t# resample_args = {\"input\" : ai_segment_path,\n",
    "\t\t\t# \t\t\t\t\t\t\t\t\t\t\t\"output\" : out_segment_path,\n",
    "\t\t\t# \t\t\t\t\t\t\t\t\t\t\t\"fixed\" : ref_image_path,#ref_image_nii_path,\n",
    "\t\t\t# \t\t\t\t\t\t\t\t\t\t\t\"interpolation\" : \"nn\"}\n",
    "\t\t\t# pypla.resample(verbose = False, **resample_args)\n",
    "\t\t\tresample_seg_with_sitk(input_seg_nifti_path=ai_segment_path,\n",
    "\t\t\t\tinput_dicom_path=out_image_dicom_for_nrrd, \n",
    "\t\t\t\toutput_seg_path=out_segment_path)\n",
    "\t\t\tshutil.copy(os.path.join(IDC_SEG_NII, serieUID_image, \"meta.json\"),\n",
    "\t\t\t\t\t\t\t\t\tos.path.join(output_serie_resampled, \"meta.json\"))\n",
    "    # combine prostate regional segments to whole prostate gland if necessary\n",
    "\t\toutput_nii_folder = os.path.join(base_idc_output_nii_folder, serieUID_image)\n",
    "\t\toutput_dicom_folder = os.path.join(base_idc_output_dicom_folder, serieUID_image)\n",
    "\t\tcombine_regions_to_prostate(input_dcmqi_nii_folder=output_serie_resampled,\n",
    "                                                                input_dcmqi_dicom_seg_path=idc_seg_dicom_path,\n",
    "                                                                input_ref_image_nii_file=ref_image_nrrd_path,#ref_image_nii_path,\n",
    "                                                                output_nii_folder=output_nii_folder,\n",
    "                                                                output_dicom_folder=output_dicom_folder,\n",
    "                                                                dicom_codeValuesProstate_lst=dicomCodeValuesProstate_lst,\n",
    "                                                                dicom_CodingSchemeDesignatorProstate_lst=dicomCodingSchemeDesignatorProstate_lst)\n",
    "\t\tdcmqi_ready_metadata_file = glob.glob(os.path.join(output_nii_folder, \"**\", \"*.json\"), recursive=True)[0]\n",
    "\t\tformatted_inputImageList = sorted(glob.glob(os.path.join(output_nii_folder, \"*.nii.gz\")))\n",
    "\t\tformatted_inputImageStr = \"','\".join(formatted_inputImageList)\n",
    "\t\toutput_dcmqi_dicom_file = os.path.join(output_dicom_folder, idc_seg_dicom_path.split(\"/\")[-1])\n",
    "\t\t!itkimage2segimage --inputImageList $formatted_inputImageStr \\\n",
    "\t\t\t--inputDICOMDirectory $out_image_dicom \\\n",
    "\t\t\t--outputDICOM $output_dcmqi_dicom_file \\\n",
    "\t\t\t--inputMetadata $dcmqi_ready_metadata_file\n",
    "\texcept:\n",
    "\t\tprint(f\"IDC COMBINATION DID NOT WORK FOR PATH : {idc_expert_serieUID}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RGDC71XpA15w",
    "outputId": "984f7f8d-bc33-4163-b54c-fa239b8763e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idc_prostate_gen/\n",
      "idc_prostate_gen/output_nii_combined/\n",
      "idc_prostate_gen/output_nii_combined/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/\n",
      "idc_prostate_gen/output_nii_combined/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/meta.json\n",
      "idc_prostate_gen/output_nii_combined/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/1.nii.gz\n",
      "idc_prostate_gen/output_nii_combined_resampled_dicom/\n",
      "idc_prostate_gen/output_nii_combined_resampled_dicom/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/\n",
      "idc_prostate_gen/output_nii_combined_resampled_dicom/1.3.6.1.4.1.14519.5.2.1.232435338427484273565021170934926902671/3f23933e-9f1a-4341-af96-199160be823d.dcm\n"
     ]
    }
   ],
   "source": [
    "# !tar -C /content/OUTPUT_EVAL -cvf - idc_prostate_gen | lz4 > radiomics_idc_output.tar.lz4"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "colab": {
   "collapsed_sections": [
    "CeyD64ZnyeAA",
    "EtveZiW5oMRY"
   ],
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}