create_w_companies.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import sklearn\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/companies.csv\").dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = companies.convert_dtypes(convert_string=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies_array = np.array(companies).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_yasemin = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/yasemin_data.csv\")\n",
    "df_yasemin_notr = df_yasemin[df_yasemin[\"label\"]==1].map(str)\n",
    "df_yasemin_neg = df_yasemin[df_yasemin[\"label\"]==2].map(str)\n",
    "df_yasemin_pos = df_yasemin[df_yasemin[\"label\"]==3].map(str)\n",
    "df_yasemin_notr[df_yasemin_notr[\"label\"]==1] = \"Notr\"\n",
    "df_yasemin_neg[df_yasemin_neg[\"label\"]==2] = \"Negative\"\n",
    "df_yasemin_pos[df_yasemin_pos[\"label\"]==3] = \"Positive\"\n",
    "df_emre = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/emre_data.csv\")\n",
    "df_emre_notr = df_emre[df_emre[\"label\"]==1].map(str)\n",
    "df_emre_neg = df_emre[df_emre[\"label\"]==2].map(str)\n",
    "df_emre_pos = df_emre[df_emre[\"label\"]==3].map(str)\n",
    "df_emre_notr[df_emre_notr[\"label\"]==1] = \"Notr\"\n",
    "df_emre_neg[df_emre_neg[\"label\"]==2] = \"Negative\"\n",
    "df_emre_pos[df_emre_pos[\"label\"]==3] = \"Positive\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = {'train': 'train.csv', 'test': 'test.csv'}\n",
    "df = pd.read_csv(\"hf://datasets/winvoker/turkish-sentiment-analysis-dataset/\" + splits[\"train\"])\n",
    "df_2 = pd.read_csv(\"hf://datasets/winvoker/turkish-sentiment-analysis-dataset/\" + splits[\"test\"])\n",
    "df.drop(\"dataset\",axis=1)\n",
    "df_2.drop(\"dataset\",axis=1)\n",
    "df_train_notr = pd.concat((df[df[\"label\"]==\"Notr\"],df_2[df_2[\"label\"]==\"Notr\"],df_yasemin_notr,df_emre_notr),ignore_index=True)[\"text\"].convert_dtypes(convert_string=True).to_list()\n",
    "df_train_pos = pd.concat((df[df[\"label\"]==\"Positive\"],df_2[df_2[\"label\"]==\"Positive\"],df_yasemin_pos,df_emre_pos),ignore_index=True)[\"text\"].convert_dtypes(convert_string=True).to_list()\n",
    "df_train_neg = pd.concat((df[df[\"label\"]==\"Negative\"],df_2[df_2[\"label\"]==\"Negative\"],df_yasemin_neg,df_emre_neg),ignore_index=True)[\"text\"].convert_dtypes(convert_string=True).to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "notr_bow = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/bows/notr_bow_not_cleared.csv\")\n",
    "positive_bow = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/bows/positive_bow_not_cleared.csv\")\n",
    "negative_bow = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/bows/negative_bow_not_cleared.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "notr_bow_set = set(notr_bow[\"words\"].map(str).to_list())\n",
    "positive_bow_set = set(positive_bow[\"words\"].map(str).to_list())\n",
    "negative_bow_set = set(negative_bow[\"words\"].map(str).to_list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_bow_set = notr_bow_set.intersection(positive_bow_set,negative_bow)\n",
    "wrong_bow_set_pos_notr = notr_bow_set.intersection(positive_bow_set)\n",
    "wrong_bow_set_pos_neg = negative_bow_set.intersection(positive_bow_set)\n",
    "wrong_bow_set_neg_notr = negative_bow_set.intersection(notr_bow_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34771\n",
      "93053\n",
      "30050\n"
     ]
    }
   ],
   "source": [
    "print(len(wrong_bow_set_pos_notr))\n",
    "print(len(wrong_bow_set_pos_neg))\n",
    "print(len(wrong_bow_set_neg_notr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "notr_bow_set -= wrong_bow_set\n",
    "positive_bow_set -= wrong_bow_set\n",
    "negative_bow_set -= wrong_bow_set\n",
    "#----------------------------------\n",
    "#positive_bow_set -= set(random.sample(list(wrong_bow_set_pos_neg),len(wrong_bow_set_pos_neg)//2))\n",
    "#----------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58347\n",
      "273831\n",
      "186740\n"
     ]
    }
   ],
   "source": [
    "print(len(notr_bow_set))\n",
    "print(len(positive_bow_set))\n",
    "print(len(negative_bow_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cities = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/cities.csv\")[\"names\"].to_list()\n",
    "counties = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/counties.csv\")[\"names\"].to_list()\n",
    "names = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/names.csv\")[\"names\"].to_list()\n",
    "surnames = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/surnames.csv\")[\"names\"].to_list()\n",
    "countries = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/countries.csv\")[\"names\"].to_list()\n",
    "places = pd.read_csv(\"/home/musasina/projects/teknofest/msnet/datasets/palces.csv\")[\"names\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/musasina/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import nltk\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "stops = set(stopwords.words(\"turkish\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_with_companies = []\n",
    "useless_pos = []\n",
    "useless_neg = []\n",
    "useless_notr = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_place_to_text(truncated_text,labels,label):\n",
    "    index_company = np.random.randint(0,len(truncated_text))\n",
    "    index_city = np.random.randint(0,len(truncated_text))\n",
    "    index_name = np.random.randint(0,len(truncated_text))\n",
    "    index_surname = np.random.randint(0,len(truncated_text))\n",
    "    index_place = np.random.randint(0,len(truncated_text))\n",
    "    index_county = np.random.randint(0,len(truncated_text))\n",
    "    index_country = np.random.randint(0,len(truncated_text))\n",
    "    #----------------------------------------------------------\n",
    "    company = str(np.random.choice(companies_array)).casefold().strip()\n",
    "    city = np.random.choice(cities).casefold().strip()\n",
    "    name = np.random.choice(names).casefold().strip()\n",
    "    surname = np.random.choice(surnames).casefold().strip()\n",
    "    place = np.random.choice(places).casefold().strip()\n",
    "    county = np.random.choice(counties).casefold().strip()\n",
    "    country = np.random.choice(countries).casefold().strip()\n",
    "    #----------------------------------------------------------\n",
    "    labels.insert(index_company,label)\n",
    "    truncated_text.insert(index_company,company)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_city,label)\n",
    "    truncated_text.insert(index_city,city)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_name,label)\n",
    "    truncated_text.insert(index_name,name)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_surname,label)\n",
    "    truncated_text.insert(index_surname,surname)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_place,label)\n",
    "    truncated_text.insert(index_place,place)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_county,label)\n",
    "    truncated_text.insert(index_county,county)\n",
    "    #-----------------------------------------\n",
    "    labels.insert(index_country,label)\n",
    "    truncated_text.insert(index_country,country)\n",
    "    return truncated_text,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_place_to_text_2(truncated_text,labels,label):\n",
    "    index_company = np.random.randint(0,len(truncated_text))\n",
    "    index_city = np.random.randint(0,len(truncated_text))\n",
    "    index_name = np.random.randint(0,len(truncated_text))\n",
    "    index_surname = np.random.randint(0,len(truncated_text))\n",
    "    index_place = np.random.randint(0,len(truncated_text))\n",
    "    index_county = np.random.randint(0,len(truncated_text))\n",
    "    index_country = np.random.randint(0,len(truncated_text))\n",
    "    #----------------------------------------------------------\n",
    "    company = str(np.random.choice(companies_array)).casefold().strip()\n",
    "    city = np.random.choice(cities).casefold().strip()\n",
    "    name = np.random.choice(names).casefold().strip()\n",
    "    surname = np.random.choice(surnames).casefold().strip()\n",
    "    place = np.random.choice(places).casefold().strip()\n",
    "    county = np.random.choice(counties).casefold().strip()\n",
    "    country = np.random.choice(countries).casefold().strip()\n",
    "    #---------------------------------------------------------\n",
    "    index = np.random.choice([index_company,index_city,index_name,index_surname,index_place,index_county,index_country])\n",
    "    named_entity = np.random.choice([company,city,name,surname,place,county,country])\n",
    "    #----------------------------------------------------------\n",
    "    labels.insert(index,label)\n",
    "    truncated_text.insert(index,named_entity)\n",
    "    #-----------------------------------------\n",
    "    return truncated_text,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data(seperated_texts:list,label,add_palces=True):\n",
    "    for truncated_text in seperated_texts:\n",
    "        labels = [0]*121\n",
    "        for i,word in enumerate(truncated_text):\n",
    "            if label == 1:\n",
    "                if word in notr_bow_set:\n",
    "                    labels[i]=1\n",
    "                if word in wrong_bow_set_pos_neg:\n",
    "                    labels[i]=1\n",
    "            elif label == 2:\n",
    "                if word in negative_bow_set:\n",
    "                    labels[i]=2\n",
    "            elif label ==3:\n",
    "                if word in positive_bow_set:\n",
    "                    labels[i]=3\n",
    "        if add_palces:\n",
    "            yield add_place_to_text(truncated_text,labels,label)\n",
    "        else:\n",
    "            yield truncated_text,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data_loop(data,sentiment):\n",
    "    useless = []\n",
    "    for text,label in zip(data,repeat(sentiment)):\n",
    "        left = 0\n",
    "        right = 121\n",
    "        seperated_texts = []\n",
    "        splitted_text = text.split()\n",
    "        count = 0\n",
    "        while left<right:\n",
    "            truncated_text = splitted_text[left:right]\n",
    "            if len(truncated_text) <121:\n",
    "                if count == 0:\n",
    "                    useless.append(\" \".join(truncated_text))\n",
    "                break\n",
    "            count += 1\n",
    "            seperated_texts.append(truncated_text)\n",
    "            left = deepcopy(right)\n",
    "            right += 121\n",
    "        for truncated_text,labels in create_data(seperated_texts,label):\n",
    "            texts_with_companies.append({\n",
    "                \"text\":\" \".join(truncated_text),\n",
    "                \"sentiment\":label,\n",
    "                \"labels\":labels\n",
    "            })\n",
    "    return useless"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dict = {\"Notr\":1,\"Negative\":2,\"Positive\":3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_useless_notr = create_data_loop(df_train_notr,1)\n",
    "df_useless_neg = create_data_loop(df_train_neg,2)\n",
    "df_useless_pos = create_data_loop(df_train_pos,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "181538\n",
      "66060\n",
      "305851\n"
     ]
    }
   ],
   "source": [
    "print(len(df_useless_notr))\n",
    "print(len(df_useless_neg))\n",
    "print(len(df_useless_pos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_useless = pd.DataFrame({\"text\":df_useless_notr+df_useless_neg+df_useless_pos,\"label\":[1]*len(df_useless_notr)+[2]*len(df_useless_neg)+[3]*len(df_useless_pos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_useless_notr = df_useless[df_useless[\"label\"]==1].sample(66060,random_state=42)\n",
    "df_useless_neg = df_useless[df_useless[\"label\"]==2].sample(66060,random_state=42)\n",
    "df_useless_pos = df_useless[df_useless[\"label\"]==3].sample(66060,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66060\n",
      "66060\n",
      "66060\n"
     ]
    }
   ],
   "source": [
    "print(len(df_useless_notr))\n",
    "print(len(df_useless_neg))\n",
    "print(len(df_useless_pos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_useless = pd.concat((df_useless_notr,df_useless_neg,df_useless_pos),ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_useless = sklearn.utils.shuffle(df_useless,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(80):\n",
    "    df_useless = sklearn.utils.shuffle(df_useless,random_state=42)\n",
    "    concated_text = []\n",
    "    labels = []\n",
    "    for text,label in zip(df_useless[\"text\"],df_useless[\"label\"]):\n",
    "        splitted_text = str(text).split()\n",
    "        if len(splitted_text) != 0:\n",
    "            for truncated_text,return_labels in create_data([splitted_text],label,add_palces=False):\n",
    "                concated_text += truncated_text[:len(splitted_text)]\n",
    "                labels += return_labels[:len(splitted_text)]\n",
    "                concated_text,labels = add_place_to_text_2(concated_text,labels,label)\n",
    "            if len(concated_text) < 128:\n",
    "                continue\n",
    "            concated_text = concated_text[:128]\n",
    "            labels = labels[:128]\n",
    "            notr_count = labels.count(1)\n",
    "            neg_count = labels.count(2)\n",
    "            pos_count = labels.count(3)\n",
    "            overall_label = 0\n",
    "            if notr_count == np.max([notr_count,neg_count,pos_count]):\n",
    "                overall_label = 1\n",
    "            elif neg_count == np.max([notr_count,neg_count,pos_count]):\n",
    "                overall_label = 2\n",
    "            else:\n",
    "                overall_label = 3\n",
    "            texts_with_companies.append({\n",
    "            \"text\":\" \".join(concated_text),\n",
    "            \"sentiment\":overall_label,\n",
    "            \"labels\":deepcopy(labels)\n",
    "            })\n",
    "            concated_text = []\n",
    "            labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.utils\n",
    "\n",
    "\n",
    "df_final = sklearn.utils.shuffle(pd.DataFrame(texts_with_companies),random_state=42)\n",
    "df_final.to_csv(\"/home/musasina/projects/teknofest/msnet/datasets/train_final_8_128.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "teknofest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}