data_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/pippertetsing/miniforge3m1/envs/llm_env/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from transformers import RobertaTokenizer\n",
    "import datasets\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "data_path = './comment_code_sol.pkl'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_pickle(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Original dataset length: 17211242\n",
      "INFO: dataset length after code length refinment: 13728139\n",
      "INFO: dataset length after comment length refinment: 12650850\n"
     ]
    }
   ],
   "source": [
    "# drop all code and comment value not longer as 20 characters\n",
    "print(\"INFO: Original dataset length:\", len(dataset))\n",
    "\n",
    "dataset=dataset[dataset['code_string'].str.len() >= 20]\n",
    "print(\"INFO: dataset length after code length refinment:\", len(dataset))\n",
    "\n",
    "dataset=dataset[dataset['comments'].str.len() >= 20]\n",
    "print(\"INFO: dataset length after comment length refinment:\", len(dataset))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: dataset length after marker processing: 12332748\n",
      "INFO: dataset length after function processing: 9901914\n",
      "INFO: dataset discarding contracts and libs: 8440378\n",
      "INFO: stripped comment!\n"
     ]
    }
   ],
   "source": [
    "# remove lines with no code\n",
    "def hasMarker(code):\n",
    "    if ';' in code:\n",
    "        return True\n",
    "    \n",
    "    elif '}' in code:\n",
    "        return True\n",
    "    \n",
    "    elif '{' in code:\n",
    "        return True\n",
    "    \n",
    "    elif ';' in code:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "\n",
    "def is_function(code):\n",
    "    return True if 'function' in code else False\n",
    "\n",
    "def discard_contract_or_lib(code):\n",
    "    if \"contract\" in code:\n",
    "        return False \n",
    "\n",
    "    if \"library\" in code:\n",
    "        return False\n",
    "    return True \n",
    "\n",
    "# remove docstring \n",
    "def strip_comment(com):\n",
    "    com = com.replace('*','').strip()\n",
    "    com = com.replace('@title','').strip()\n",
    "    com = com.replace('@author','').strip()\n",
    "    com = com.replace('@notice','').strip()\n",
    "    com = com.replace('@dev','').strip()\n",
    "    com = com.replace('@param','').strip()\n",
    "    com = com.replace('#','').strip()\n",
    "    com = com.replace('@return','return').strip()\n",
    "    return com\n",
    "\n",
    "dataset=dataset[dataset['code_string'].apply(lambda x: hasMarker(x))]\n",
    "print(\"INFO: dataset length after marker processing:\", len(dataset))\n",
    "\n",
    "dataset=dataset[dataset['code_string'].apply(lambda x: is_function(x))]\n",
    "print(\"INFO: dataset length after function processing:\", len(dataset))\n",
    "\n",
    "dataset=dataset[dataset['code_string'].apply(lambda x: discard_contract_or_lib(x))]\n",
    "print(\"INFO: dataset discarding contracts and libs:\", len(dataset))\n",
    "\n",
    "dataset['comments']=dataset['comments'].apply(lambda x: strip_comment(x))\n",
    "print(\"INFO: stripped comment!\")\n",
    " \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: dataset length after dropping duplicates: 739927\n"
     ]
    }
   ],
   "source": [
    "# Drop duplicate on code value\n",
    "dataset = dataset.drop_duplicates(subset=['code_string'], keep='first')\n",
    "print(\"INFO: dataset length after dropping duplicates:\", len(dataset))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>comments</th>\n",
       "      <th>code_string</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>// SPDX-License-Identifier: MIT\\npragma solidi...</td>\n",
       "      <td>Withdraw ether from this contract (Callable by...</td>\n",
       "      <td>function withdraw() onlyOwner public {\\n  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>// SPDX-License-Identifier: MIT\\npragma solidi...</td>\n",
       "      <td>_setTokenURI(newTokenId, Strings.toString(newT...</td>\n",
       "      <td>function steamDumplings(uint256 numDumplin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>// SPDX-License-Identifier: MIT\\npragma solidi...</td>\n",
       "      <td>See {IERC721-balanceOf}./</td>\n",
       "      <td>function balanceOf(address owner) public v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>// SPDX-License-Identifier: MIT\\npragma solidi...</td>\n",
       "      <td>See {IERC721-ownerOf}./</td>\n",
       "      <td>function ownerOf(uint256 tokenId) public v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>// SPDX-License-Identifier: MIT\\npragma solidi...</td>\n",
       "      <td>See {IERC721Metadata-name}./</td>\n",
       "      <td>function name() public view override retur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17211114</th>\n",
       "      <td>./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...</td>\n",
       "      <td>Upgrades target  newTarget New target  newTarg...</td>\n",
       "      <td>function upgradeTarget(address newTarget, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17211115</th>\n",
       "      <td>./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...</td>\n",
       "      <td>Performs a delegatecall to the contract implem...</td>\n",
       "      <td>function() external payable {\\n        add...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17211120</th>\n",
       "      <td>./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...</td>\n",
       "      <td>Notice period before activation preparation st...</td>\n",
       "      <td>function getNoticePeriod() external return...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17211125</th>\n",
       "      <td>./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...</td>\n",
       "      <td>Checks that contract is ready for upgrade retu...</td>\n",
       "      <td>function isReadyForUpgrade() external retu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17211233</th>\n",
       "      <td>./partial_match/77/0x925aFFbA0902cC94d103a1CF5...</td>\n",
       "      <td>Give 'voter' the right to vote on this ballot....</td>\n",
       "      <td>function giveRightToVote___(address voter)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>739927 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  file_name  \\\n",
       "2         // SPDX-License-Identifier: MIT\\npragma solidi...   \n",
       "3         // SPDX-License-Identifier: MIT\\npragma solidi...   \n",
       "9         // SPDX-License-Identifier: MIT\\npragma solidi...   \n",
       "10        // SPDX-License-Identifier: MIT\\npragma solidi...   \n",
       "11        // SPDX-License-Identifier: MIT\\npragma solidi...   \n",
       "...                                                     ...   \n",
       "17211114  ./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...   \n",
       "17211115  ./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...   \n",
       "17211120  ./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...   \n",
       "17211125  ./partial_match/77/0xa846788E1D9aB3F90f8bb9b1B...   \n",
       "17211233  ./partial_match/77/0x925aFFbA0902cC94d103a1CF5...   \n",
       "\n",
       "                                                   comments  \\\n",
       "2         Withdraw ether from this contract (Callable by...   \n",
       "3         _setTokenURI(newTokenId, Strings.toString(newT...   \n",
       "9                                 See {IERC721-balanceOf}./   \n",
       "10                                  See {IERC721-ownerOf}./   \n",
       "11                             See {IERC721Metadata-name}./   \n",
       "...                                                     ...   \n",
       "17211114  Upgrades target  newTarget New target  newTarg...   \n",
       "17211115  Performs a delegatecall to the contract implem...   \n",
       "17211120  Notice period before activation preparation st...   \n",
       "17211125  Checks that contract is ready for upgrade retu...   \n",
       "17211233  Give 'voter' the right to vote on this ballot....   \n",
       "\n",
       "                                                code_string  \n",
       "2             function withdraw() onlyOwner public {\\n  ...  \n",
       "3             function steamDumplings(uint256 numDumplin...  \n",
       "9             function balanceOf(address owner) public v...  \n",
       "10            function ownerOf(uint256 tokenId) public v...  \n",
       "11            function name() public view override retur...  \n",
       "...                                                     ...  \n",
       "17211114      function upgradeTarget(address newTarget, ...  \n",
       "17211115      function() external payable {\\n        add...  \n",
       "17211120      function getNoticePeriod() external return...  \n",
       "17211125      function isReadyForUpgrade() external retu...  \n",
       "17211233      function giveRightToVote___(address voter)...  \n",
       "\n",
       "[739927 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_dataset = Dataset.from_pandas(dataset)\n",
    "hf_dataset = hf_dataset.train_test_split(test_size=0.2, seed=100)\n",
    "test_valid = hf_dataset['test'].train_test_split(test_size=0.5, seed=100)\n",
    "\n",
    "Hdataset = DatasetDict({\n",
    "            'train': hf_dataset['train'],\n",
    "            'test': test_valid['test'],\n",
    "            'valid': test_valid['train']\n",
    "            })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['file_name', 'comments', 'code_string', '__index_level_0__'],\n",
       "        num_rows: 591941\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['file_name', 'comments', 'code_string', '__index_level_0__'],\n",
       "        num_rows: 73993\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['file_name', 'comments', 'code_string', '__index_level_0__'],\n",
       "        num_rows: 73993\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hdataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Pushing split train to the Hub.\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:02<00:00, 13.30ba/s]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.53ba/s]74s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.60ba/s]71s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.92ba/s]94s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:05<00:00,  6.18ba/s]16s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:05<00:00,  6.18ba/s]18s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.93ba/s]23s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.54ba/s]16s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.55ba/s].03s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.29ba/s].95s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:07<00:00,  5.28ba/s]70.13s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:07<00:00,  5.00ba/s]55.22s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.36ba/s]25.11s/it]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.35ba/s]02.75s/it]\n",
      "Pushing dataset shards to the dataset hub: 100%|██████████| 16/16 [1:21:36<00:00, 306.03s/it]\n",
      "Pushing split test to the Hub.\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.51ba/s]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.70ba/s]1s/it]\n",
      "Pushing dataset shards to the dataset hub: 100%|██████████| 2/2 [10:54<00:00, 327.16s/it]\n",
      "Pushing split valid to the Hub.\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.64ba/s]\n",
      "Creating parquet from Arrow format: 100%|██████████| 37/37 [00:06<00:00,  5.42ba/s]3s/it]\n",
      "Pushing dataset shards to the dataset hub: 100%|██████████| 2/2 [19:27<00:00, 583.67s/it]\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "Hdataset.push_to_hub(\"Pipper/SolFuncs\",  token=os.environ.get(\"HF_TOKEN\"), max_shard_size=\"1GB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 6\n",
    "print(dataset.comments.iloc[i])\n",
    "print('-'*100)\n",
    "print(dataset.code_string.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.to_pickle('./filtered_comment_code_sol.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}