feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa\n",
    "import librosa.display\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "RAV = r\"C:\\Users\\hp\\Desktop\\speech_emotion\\Ravdess\"\n",
    "SAVEE = r\"C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\"\n",
    "TESS = r\"C:\\Users\\hp\\Desktop\\speech_emotion\\TESS Toronto emotional speech set data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male_neutral     120\n",
       "male_angry        60\n",
       "male_disgust      60\n",
       "male_fear         60\n",
       "male_happy        60\n",
       "male_sad          60\n",
       "male_surprise     60\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir_list = os.listdir(SAVEE)\n",
    "emotion=[]\n",
    "path = []\n",
    "for i in dir_list:\n",
    "    if i[-8:-6]=='_a':\n",
    "        emotion.append('male_angry')\n",
    "    elif i[-8:-6]=='_d':\n",
    "        emotion.append('male_disgust')\n",
    "    elif i[-8:-6]=='_f':\n",
    "        emotion.append('male_fear')\n",
    "    elif i[-8:-6]=='_h':\n",
    "        emotion.append('male_happy')\n",
    "    elif i[-8:-6]=='_n':\n",
    "        emotion.append('male_neutral')\n",
    "    elif i[-8:-6]=='sa':\n",
    "        emotion.append('male_sad')\n",
    "    elif i[-8:-6]=='su':\n",
    "        emotion.append('male_surprise')\n",
    "    else:\n",
    "        emotion.append('male_error') \n",
    "    path.append(SAVEE + \"\\\\\"+i)\n",
    "\n",
    "SAVEE_df = pd.DataFrame(emotion, columns = ['labels'])\n",
    "SAVEE_df['source'] = 'SAVEE'\n",
    "SAVEE_df = pd.concat([SAVEE_df, pd.DataFrame(path, columns = ['path'])], axis = 1)\n",
    "SAVEE_df.labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male_neutral       144\n",
       "female_neutral     144\n",
       "male_happy          96\n",
       "male_sad            96\n",
       "male_angry          96\n",
       "male_fear           96\n",
       "male_disgust        96\n",
       "male_surprise       96\n",
       "female_happy        96\n",
       "female_sad          96\n",
       "female_angry        96\n",
       "female_fear         96\n",
       "female_disgust      96\n",
       "female_surprise     96\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir_list = os.listdir(RAV)\n",
    "dir_list.sort()\n",
    "\n",
    "emotion = []\n",
    "gender = []\n",
    "path = []\n",
    "for i in dir_list:\n",
    "    fname = os.listdir(RAV +\"\\\\\"+ i)\n",
    "    for f in fname:\n",
    "        part = f.split('.')[0].split('-')\n",
    "        emotion.append(int(part[2]))\n",
    "        temp = int(part[6])\n",
    "        if temp%2 == 0:\n",
    "            temp = \"female\"\n",
    "        else:\n",
    "            temp = \"male\"\n",
    "        gender.append(temp)\n",
    "        path.append(RAV +\"\\\\\"+ i + '\\\\' + f)\n",
    "\n",
    "        \n",
    "RAV_df = pd.DataFrame(emotion)\n",
    "RAV_df = RAV_df.replace({1:'neutral', 2:'neutral', 3:'happy', 4:'sad', 5:'angry', 6:'fear', 7:'disgust', 8:'surprise'})\n",
    "RAV_df = pd.concat([pd.DataFrame(gender),RAV_df],axis=1)\n",
    "RAV_df.columns = ['gender','emotion']\n",
    "RAV_df['labels'] =RAV_df.gender + '_' + RAV_df.emotion\n",
    "RAV_df['source'] = 'RAVDESS'  \n",
    "RAV_df = pd.concat([RAV_df,pd.DataFrame(path, columns = ['path'])],axis=1)\n",
    "RAV_df = RAV_df.drop(['gender', 'emotion'], axis=1)\n",
    "RAV_df.labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "EmotionData_temp = pd.concat([SAVEE_df, RAV_df], axis = 0)\n",
    "#print(EmotionData.labels.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hp\\Desktop\\speech_emotion\\TESS Toronto emotional speech set data\n",
      "OAF_angry\n",
      "OAF_disgust\n",
      "OAF_Fear\n",
      "OAF_happy\n",
      "OAF_neutral\n",
      "OAF_Pleasant_surprise\n",
      "OAF_Sad\n",
      "YAF_angry\n",
      "YAF_disgust\n",
      "YAF_fear\n",
      "YAF_happy\n",
      "YAF_neutral\n",
      "YAF_pleasant_surprised\n",
      "YAF_sad\n",
      "(2800, 3)\n"
     ]
    }
   ],
   "source": [
    "tess_directory_list=os.listdir(TESS)\n",
    "\n",
    "file_emotion = []\n",
    "file_path = []\n",
    "print(TESS)\n",
    "for dir in tess_directory_list:\n",
    "    print(dir)\n",
    "    directories = os.listdir(TESS +\"\\\\\"+ dir)\n",
    "    for file in directories:\n",
    "        #print(file)\n",
    "        part = file.split('.')[0]\n",
    "        part = part.split('_')[-1]\n",
    "        if part=='ps':\n",
    "            file_emotion.append('female_surprise')\n",
    "        elif(part==\"surprised\"):\n",
    "            file_emotion.append('female_surprise')\n",
    "        elif(part==\"fear\"):\n",
    "            file_emotion.append('female_fear')\n",
    "        elif(part==\"sad\"):\n",
    "            file_emotion.append('female_sad')\n",
    "        elif(part==\"angry\"):\n",
    "            file_emotion.append('female_angry')\n",
    "        elif(part==\"disgust\"):\n",
    "            file_emotion.append('female_disgust')\n",
    "        elif(part==\"nuetral\"):\n",
    "            file_emotion.append('female_neutral')\n",
    "        else:\n",
    "            file_emotion.append(\"female_\" + part)\n",
    "        file_path.append(TESS +\"\\\\\"+ dir + '\\\\' + file)\n",
    "            \n",
    "# dataframe for emotion of files\n",
    "emotion_df = pd.DataFrame(file_emotion, columns=['labels'])\n",
    "# dataframe for path of files.\n",
    "path_df = pd.DataFrame (file_path, columns=['path'])\n",
    "Tess_df = pd.concat ([emotion_df, path_df], axis=1)\n",
    "Tess_df['source'] = 'TESS'\n",
    "Tess_df.head()\n",
    "\n",
    "print(np.shape(Tess_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "female_neutral     544\n",
      "female_happy       496\n",
      "female_sad         496\n",
      "female_angry       496\n",
      "female_fear        496\n",
      "female_disgust     496\n",
      "female_surprise    496\n",
      "male_neutral       264\n",
      "male_angry         156\n",
      "male_disgust       156\n",
      "male_fear          156\n",
      "male_happy         156\n",
      "male_sad           156\n",
      "male_surprise      156\n",
      "Name: labels, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "EmotionData = pd.concat([EmotionData_temp, Tess_df], axis = 0)\n",
    "print(EmotionData.labels.value_counts())\n",
    "\n",
    "EmotionData.to_csv(\"Data_path.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4720, 3)\n"
     ]
    }
   ],
   "source": [
    "ref=pd.read_csv(\"Data_path.csv\")\n",
    "print (np.shape(ref))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4720\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-137.28888, 111.72388, -21.969687, -20.997446...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[-109.47295, 67.51979, -40.233547, -5.063055, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-132.01605, 88.65481, -32.769714, -9.400137, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-103.55519, 83.91427, -34.984425, -17.377562,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[-143.46548, 69.09458, -28.198627, -13.793649,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             feature\n",
       "0  [-137.28888, 111.72388, -21.969687, -20.997446...\n",
       "1  [-109.47295, 67.51979, -40.233547, -5.063055, ...\n",
       "2  [-132.01605, 88.65481, -32.769714, -9.400137, ...\n",
       "3  [-103.55519, 83.91427, -34.984425, -17.377562,...\n",
       "4  [-143.46548, 69.09458, -28.198627, -13.793649,..."
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns=['feature'])\n",
    "\n",
    "counter=0\n",
    "\n",
    "for index,path in enumerate(ref.path):\n",
    "    source=ref.source[index]\n",
    "    if source==\"RAVDESS\":\n",
    "        d=2.5\n",
    "        o=0.5\n",
    "    if source==\"SAVEE\":\n",
    "        d=5.5\n",
    "        o=0.5\n",
    "    if source==\"TESS\":\n",
    "        d=1.5\n",
    "        o=0.0\n",
    " \n",
    "    X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=d,sr=4410,offset=o)\n",
    "    sample_rate = np.array(sample_rate)\n",
    "    mfcc=librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)\n",
    "    mfcc_m=np.mean(mfcc,axis=1)  \n",
    "\n",
    "    delta=librosa.feature.delta(mfcc)\n",
    "    delta_m=np.mean(delta,axis=1)\n",
    "    \n",
    "    delta_delta=librosa.feature.delta(delta)\n",
    "    delta_delta_m=np.mean(delta_delta,axis=1)\n",
    "    \n",
    "    feature = np.concatenate((mfcc_m, delta_m,delta_delta_m))\n",
    "\n",
    "    df.loc[counter] = [feature]\n",
    "    counter=counter+1   \n",
    "\n",
    "\n",
    "print(len(df))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([ref,pd.DataFrame(df['feature'].values.tolist())],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>source</th>\n",
       "      <th>path</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>...</th>\n",
       "      <th>29</th>\n",
       "      <th>30</th>\n",
       "      <th>31</th>\n",
       "      <th>32</th>\n",
       "      <th>33</th>\n",
       "      <th>34</th>\n",
       "      <th>35</th>\n",
       "      <th>36</th>\n",
       "      <th>37</th>\n",
       "      <th>38</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>male_angry</td>\n",
       "      <td>SAVEE</td>\n",
       "      <td>C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...</td>\n",
       "      <td>-137.288879</td>\n",
       "      <td>111.723877</td>\n",
       "      <td>-21.969687</td>\n",
       "      <td>-20.997446</td>\n",
       "      <td>-48.871670</td>\n",
       "      <td>-2.439504</td>\n",
       "      <td>-34.307919</td>\n",
       "      <td>...</td>\n",
       "      <td>0.525392</td>\n",
       "      <td>0.662771</td>\n",
       "      <td>0.348220</td>\n",
       "      <td>0.523438</td>\n",
       "      <td>0.086971</td>\n",
       "      <td>0.450639</td>\n",
       "      <td>-0.046797</td>\n",
       "      <td>0.287537</td>\n",
       "      <td>0.018269</td>\n",
       "      <td>0.103508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>male_angry</td>\n",
       "      <td>SAVEE</td>\n",
       "      <td>C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...</td>\n",
       "      <td>-109.472954</td>\n",
       "      <td>67.519791</td>\n",
       "      <td>-40.233547</td>\n",
       "      <td>-5.063055</td>\n",
       "      <td>-39.719791</td>\n",
       "      <td>7.764367</td>\n",
       "      <td>-25.280474</td>\n",
       "      <td>...</td>\n",
       "      <td>0.489980</td>\n",
       "      <td>0.170700</td>\n",
       "      <td>-0.234358</td>\n",
       "      <td>0.578445</td>\n",
       "      <td>-0.081931</td>\n",
       "      <td>0.415396</td>\n",
       "      <td>0.141780</td>\n",
       "      <td>0.291497</td>\n",
       "      <td>-0.048786</td>\n",
       "      <td>0.022573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>male_angry</td>\n",
       "      <td>SAVEE</td>\n",
       "      <td>C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...</td>\n",
       "      <td>-132.016052</td>\n",
       "      <td>88.654808</td>\n",
       "      <td>-32.769714</td>\n",
       "      <td>-9.400137</td>\n",
       "      <td>-35.487514</td>\n",
       "      <td>4.695376</td>\n",
       "      <td>-29.375397</td>\n",
       "      <td>...</td>\n",
       "      <td>1.133303</td>\n",
       "      <td>1.217249</td>\n",
       "      <td>-0.244995</td>\n",
       "      <td>0.965961</td>\n",
       "      <td>0.180155</td>\n",
       "      <td>0.346001</td>\n",
       "      <td>0.007635</td>\n",
       "      <td>0.802352</td>\n",
       "      <td>-0.099798</td>\n",
       "      <td>0.037451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>male_angry</td>\n",
       "      <td>SAVEE</td>\n",
       "      <td>C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...</td>\n",
       "      <td>-103.555191</td>\n",
       "      <td>83.914268</td>\n",
       "      <td>-34.984425</td>\n",
       "      <td>-17.377562</td>\n",
       "      <td>-56.287861</td>\n",
       "      <td>7.178759</td>\n",
       "      <td>-29.905863</td>\n",
       "      <td>...</td>\n",
       "      <td>0.715701</td>\n",
       "      <td>0.453319</td>\n",
       "      <td>0.138438</td>\n",
       "      <td>0.757691</td>\n",
       "      <td>0.041856</td>\n",
       "      <td>0.765676</td>\n",
       "      <td>0.114095</td>\n",
       "      <td>0.511521</td>\n",
       "      <td>-0.135596</td>\n",
       "      <td>0.007118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>male_angry</td>\n",
       "      <td>SAVEE</td>\n",
       "      <td>C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...</td>\n",
       "      <td>-143.465485</td>\n",
       "      <td>69.094582</td>\n",
       "      <td>-28.198627</td>\n",
       "      <td>-13.793649</td>\n",
       "      <td>-44.961941</td>\n",
       "      <td>-1.339370</td>\n",
       "      <td>-27.697914</td>\n",
       "      <td>...</td>\n",
       "      <td>0.685708</td>\n",
       "      <td>0.871115</td>\n",
       "      <td>0.040399</td>\n",
       "      <td>0.495775</td>\n",
       "      <td>-0.230098</td>\n",
       "      <td>0.320866</td>\n",
       "      <td>-0.022060</td>\n",
       "      <td>0.349066</td>\n",
       "      <td>0.242089</td>\n",
       "      <td>0.206005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       labels source                                               path  \\\n",
       "0  male_angry  SAVEE  C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...   \n",
       "1  male_angry  SAVEE  C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...   \n",
       "2  male_angry  SAVEE  C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...   \n",
       "3  male_angry  SAVEE  C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...   \n",
       "4  male_angry  SAVEE  C:\\Users\\hp\\Desktop\\speech_emotion\\Savee\\DC_a0...   \n",
       "\n",
       "            0           1          2          3          4         5  \\\n",
       "0 -137.288879  111.723877 -21.969687 -20.997446 -48.871670 -2.439504   \n",
       "1 -109.472954   67.519791 -40.233547  -5.063055 -39.719791  7.764367   \n",
       "2 -132.016052   88.654808 -32.769714  -9.400137 -35.487514  4.695376   \n",
       "3 -103.555191   83.914268 -34.984425 -17.377562 -56.287861  7.178759   \n",
       "4 -143.465485   69.094582 -28.198627 -13.793649 -44.961941 -1.339370   \n",
       "\n",
       "           6  ...        29        30        31        32        33        34  \\\n",
       "0 -34.307919  ...  0.525392  0.662771  0.348220  0.523438  0.086971  0.450639   \n",
       "1 -25.280474  ...  0.489980  0.170700 -0.234358  0.578445 -0.081931  0.415396   \n",
       "2 -29.375397  ...  1.133303  1.217249 -0.244995  0.965961  0.180155  0.346001   \n",
       "3 -29.905863  ...  0.715701  0.453319  0.138438  0.757691  0.041856  0.765676   \n",
       "4 -27.697914  ...  0.685708  0.871115  0.040399  0.495775 -0.230098  0.320866   \n",
       "\n",
       "         35        36        37        38  \n",
       "0 -0.046797  0.287537  0.018269  0.103508  \n",
       "1  0.141780  0.291497 -0.048786  0.022573  \n",
       "2  0.007635  0.802352 -0.099798  0.037451  \n",
       "3  0.114095  0.511521 -0.135596  0.007118  \n",
       "4 -0.022060  0.349066  0.242089  0.206005  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}