Document_split.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"mount_file_id":"1B81e9PiPmlpS2hAK1BKBGSLUWCL1yuEB","authorship_tag":"ABX9TyMhW+0k15FCCE/NfdzoON4Z"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"BAYgH6U78iut","executionInfo":{"status":"ok","timestamp":1714389526233,"user_tz":-60,"elapsed":911,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"7d385441-0d4f-4a8e-c578-29044aad1471"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["   document                                          full_text  \\\n","0         7  Design Thinking for innovation reflexion-Avril...   \n","1        10  Diego Estrada\\n\\nDesign Thinking Assignment\\n\\...   \n","2        16  Reporting process\\n\\nby Gilberto Gamboa\\n\\nCha...   \n","3        20  Design Thinking for Innovation\\n\\nSindy Samaca...   \n","4        56  Assignment:  Visualization Reflection  Submitt...   \n","\n","                                              tokens  \\\n","0  [Design, Thinking, for, innovation, reflexion,...   \n","1  [Diego, Estrada, \\n\\n, Design, Thinking, Assig...   \n","2  [Reporting, process, \\n\\n, by, Gilberto, Gambo...   \n","3  [Design, Thinking, for, Innovation, \\n\\n, Sind...   \n","4  [Assignment, :,   , Visualization,  , Reflecti...   \n","\n","                                 trailing_whitespace  \n","0  [True, True, True, True, False, False, True, F...  \n","1  [True, False, False, True, True, False, False,...  \n","2  [True, False, False, True, True, False, False,...  \n","3  [True, True, True, False, False, True, False, ...  \n","4  [False, False, False, False, False, False, Fal...  "],"text/html":["\n","  <div id=\"df-c81e6a9f-4bbf-4e97-be37-304dbfa8dd9f\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>document</th>\n","      <th>full_text</th>\n","      <th>tokens</th>\n","      <th>trailing_whitespace</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>7</td>\n","      <td>Design Thinking for innovation reflexion-Avril...</td>\n","      <td>[Design, Thinking, for, innovation, reflexion,...</td>\n","      <td>[True, True, True, True, False, False, True, F...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>10</td>\n","      <td>Diego Estrada\\n\\nDesign Thinking Assignment\\n\\...</td>\n","      <td>[Diego, Estrada, \\n\\n, Design, Thinking, Assig...</td>\n","      <td>[True, False, False, True, True, False, False,...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>16</td>\n","      <td>Reporting process\\n\\nby Gilberto Gamboa\\n\\nCha...</td>\n","      <td>[Reporting, process, \\n\\n, by, Gilberto, Gambo...</td>\n","      <td>[True, False, False, True, True, False, False,...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>20</td>\n","      <td>Design Thinking for Innovation\\n\\nSindy Samaca...</td>\n","      <td>[Design, Thinking, for, Innovation, \\n\\n, Sind...</td>\n","      <td>[True, True, True, False, False, True, False, ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>56</td>\n","      <td>Assignment:  Visualization Reflection  Submitt...</td>\n","      <td>[Assignment, :,   , Visualization,  , Reflecti...</td>\n","      <td>[False, False, False, False, False, False, Fal...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c81e6a9f-4bbf-4e97-be37-304dbfa8dd9f')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-c81e6a9f-4bbf-4e97-be37-304dbfa8dd9f button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-c81e6a9f-4bbf-4e97-be37-304dbfa8dd9f');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-7f0a2c56-ef98-49b4-acc4-044e54cb61cb\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7f0a2c56-ef98-49b4-acc4-044e54cb61cb')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-7f0a2c56-ef98-49b4-acc4-044e54cb61cb button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"train_df","summary":"{\n  \"name\": \"train_df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"document\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 46,\n        \"min\": 7,\n        \"max\": 123,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          112,\n          10,\n          86\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"full_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"Reflection \\u2013 Learning Launch\\n\\nFrancisco Ferreira\\n\\nChallenge\\n\\nI take part of a social enterprenuership group in my university. We were in contact\\n\\nwith Cap\\u00e3o das Antas, a rural Community in the suburbs of S\\u00e3o Carlos (SP-Brazil). We had the  intention to criate, with the local producers, a project that wold improve the quality of life in  the neighborhood. For that, in the first two months we begin to listen each farmer of Cap\\u00e3o  and try identify the principal issues of the place and their residentes. After some visits we  concluded that the principal problem was selling the products that they plant in the local  farms. Therefore, we had the objective to make and test a business model that would increase  their rent.\\n\\nSelection\\n\\nIn order to build a sustainable business model with the farmers, we made a first test\\n\\nwere we delivered the products at the costumer\\u2019s door. The action had a good result but we  want to increase even more ours sales. So we used Learning Launchs to test new ideais and  then improve our business model.\\n\\nApplication\\n\\nAfter we analise all the data we had and build the business model, Its was far from be\\n\\nthe most sustainable and profitable one. To improve our project, we made a plan to test new  ways of seling. So every week, after we analise all the process from the farm to the costumer  door, we propose and teste the improvement in the current business model.\\n\\nIn order to give examples of how it works, after some weeks we notice that the\\n\\nfarmers were having problems in separate the orders. The costumers were ordering with na  app called whatsapp, but the person who was responsable for taking note of the orders was  messy with the conversations feed. As soon as we notice this problem we proposed a new app  that automatically cluster the orders in a simple feed. Therefore we tested this improvement  in the next week and got zero mistakes in separating the orders.\\n\\nInsights\\n\\nThe application of Learning Launches showed me how testing is importante in building\\n\\na solid business model. Therefore, I learned that the more you test possibles ideias, more  chance of sucess you have. Testing a lot of improvement and ideas is not the fastest way to  achieve a better scenario but is cleary a certainly strategy to get there.\\n\\nApproach\\n\\nDuring all the improvement plan that we act every week we propose changes based on\\n\\nwhat we were seeing during the deliveries. I think that we could have achieved better results  by analysis more sources of data, like feedback from all the stakeholders. Besides that we  could have made a better knowledge management of the changes proposed for the business  model.\\n\\n\",\n          \"Diego Estrada\\n\\nDesign Thinking Assignment\\n\\nVisualization Tool\\n\\nChallenge & Selection\\n\\nThe elderly were having a hard time adapting to the changes we brought in our bank. As  a result of a poorly implemented linear solution, a more customer centric approach was  needed.\\n\\nAfter learning about design thinking in this course, we decided to apply it to solve this  problem. The visualization tool allowed the team to create a dynamic presentation using  diagrams, figures and drawings on the go that really resonated among the stakeholders.  Previous to this change, none of our solutions seemed to be adequate for them, but the  new implementation created a different type of connection with them that helped them  understand the problem in the way the team and I did.\\n\\nApplication\\n\\nThe process starts in the prep time. The team uses a series of tools and software to  develop a presentation using the surveys gathered during research and the solutions we  created during the process. The use of graphs to quickly show statistics in a fully visual  way, rather than verbally was a game changer.\\n\\nAfter having a presentation prepared, the team hands an activity to the stakeholders,  where the solutions discussed previously appear. Nonetheless, the solutions need more  work to them. After this. The stakeholders are asked to help complete the solutions  while the team and I create diagrams on a blackboard to represent how their  suggestions would impact on this specific problem.\\n\\nThe use of a group activity strengthens the bond between the company and their  investors. It makes them feel like they take part and help solve the problems as well as  show how customer centric the solutions are. Every complaint and suggestion from  customers are read and evaluated using the graph shown in the course (Involving: can  we do it? Can we afford it? \\u2026). The finalization of this activity leaves the team and the  stakeholders on the same page. It allows them to completely understand and feel part  of the solution and also gives them the chance to ask better questions, which eases the  work of the team.\\n\\nInsight & Approach\\n\\nThe use of this method created a new workflow in the Design Team. It increased the  productivity and the success rate as well as the customer/stakeholders satisfaction. The  use of the visualization tool created an engaged group of people who work together to\\n\\nDiego Estrada\\n\\nfind a solution based on their customer satisfaction. This solution is later revised and  tweaked with the help of the stakeholders who are deeply involved in the process.\\n\\nPresentations, graphics, and activities have added a huge increase in satisfaction. As a  company we also learnt that engaging different areas can be difficult because of the  varying levels of understanding, but when paired with the adequate process things just  flow.\\n\\n(This story is fictional and was created for solving the assignment)\\n\\n\",\n          \"Cheese Startup - Learning Launch \\u200bby Eladio Amaya\\n\\nChallenge\\n\\nWe are a small company in Barcelona (Spain) that refines high-quality organic cheeses  from small producers in the Pyrenees. We also trade milk and yogurts from these  producers. We distribute our dairy products mainly to haute cuisine restaurants (60% of  our turnover) and to Delicatesens stores (35% of our turnover), the remaining 5% we  distribute to individuals, mainly family, friends and acquaintances.    Selection\\n\\nI have chosen the \\\"Learning Launch\\\" because it is more adapted to my way of working  and the needs of the company.    Application\\n\\nDue to the coronavirus crisis, all restaurants have closed and Delicatesen store sales  have dropped dramatically. therefore we have had to change our target niche and head  towards private individuals and common stores.    We have carried out a survey both in small stores and among individuals. In the survey  we asked what types of cheese are the most willing to consume, as well as milk and  yogurts, also the type of milk (goat, cow, sheep), the level of curing and the price they  would be willing to pay for our products. We have also proposed sending boxes with  different products on offer, in which the average price is significantly lower.    After the survey and the observations of individuals and small stores, we have focused  our offer on a certain type of product, we have distributed it and analyzed the result not  only with our customers but also with objective sales figures.    Insight\\n\\nThe result has been spectacular, our weekly turnover had fallen to 25% with a loss of  75% (60% of restaurants and 15% of delicatessen stores). The situation was dramatic  but nowadays sales to individuals have risen from 5% to 25% and common food stores  have gone from 0% to 15%.\\n\\nApproach\\n\\nOur situation is not good, but we hope that when the restaurants open and the economy  recovers. We will have a much broader portfolio of clients and a way of working based  on the client and transferred to the producer. Before, we used to transfer products from  small producers to customers. Now the flow has changed.\\n\\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tokens\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trailing_whitespace\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":1}],"source":["import pandas as pd\n","\n","json_file_path = '/content/drive/MyDrive/Colab Notebooks/Kaggle competition /PII_detection/DeBERTa/data/test.json'\n","train_df = pd.read_json(json_file_path)\n","\n","train_df.head()"]},{"cell_type":"code","source":["cols_to_drop = ['full_text', 'trailing_whitespace']\n","train_df = train_df.drop(columns=cols_to_drop, axis=1)\n","train_df"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":363},"id":"W6dCYGtp885b","executionInfo":{"status":"ok","timestamp":1714389531886,"user_tz":-60,"elapsed":274,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"b8a849b8-5db5-4ac5-8145-461e67b167a1"},"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   document                                             tokens\n","0         7  [Design, Thinking, for, innovation, reflexion,...\n","1        10  [Diego, Estrada, \\n\\n, Design, Thinking, Assig...\n","2        16  [Reporting, process, \\n\\n, by, Gilberto, Gambo...\n","3        20  [Design, Thinking, for, Innovation, \\n\\n, Sind...\n","4        56  [Assignment, :,   , Visualization,  , Reflecti...\n","5        86  [Cheese, Startup, -, Learning, Launch, ​by, El...\n","6        93  [Silvia, Villalobos, \\n\\n, Challenge, :, \\n\\n,...\n","7       104  [Storytelling,  , The, Path, to, Innovation, \\...\n","8       112  [Reflection, –, Learning, Launch, \\n\\n, Franci...\n","9       123  [Gandhi, Institute, of, Technology, and, Manag..."],"text/html":["\n","  <div id=\"df-3d95dbd2-a745-4445-92d1-72323ad36cfd\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>document</th>\n","      <th>tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>7</td>\n","      <td>[Design, Thinking, for, innovation, reflexion,...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>10</td>\n","      <td>[Diego, Estrada, \\n\\n, Design, Thinking, Assig...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>16</td>\n","      <td>[Reporting, process, \\n\\n, by, Gilberto, Gambo...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>20</td>\n","      <td>[Design, Thinking, for, Innovation, \\n\\n, Sind...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>56</td>\n","      <td>[Assignment, :,   , Visualization,  , Reflecti...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>86</td>\n","      <td>[Cheese, Startup, -, Learning, Launch, ​by, El...</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>93</td>\n","      <td>[Silvia, Villalobos, \\n\\n, Challenge, :, \\n\\n,...</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>104</td>\n","      <td>[Storytelling,  , The, Path, to, Innovation, \\...</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>112</td>\n","      <td>[Reflection, –, Learning, Launch, \\n\\n, Franci...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>123</td>\n","      <td>[Gandhi, Institute, of, Technology, and, Manag...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d95dbd2-a745-4445-92d1-72323ad36cfd')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-3d95dbd2-a745-4445-92d1-72323ad36cfd button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-3d95dbd2-a745-4445-92d1-72323ad36cfd');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-40b93857-f71d-48f4-8669-72c8739000d0\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-40b93857-f71d-48f4-8669-72c8739000d0')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-40b93857-f71d-48f4-8669-72c8739000d0 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"train_df","summary":"{\n  \"name\": \"train_df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"document\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 46,\n        \"min\": 7,\n        \"max\": 123,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          112,\n          10,\n          86\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tokens\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":2}]},{"cell_type":"code","source":["# create len tokens column\n","train_df['number of tokens'] = train_df.tokens.apply(len)\n","train_df"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":363},"id":"nD1zhR1S9JhW","executionInfo":{"status":"ok","timestamp":1714389573485,"user_tz":-60,"elapsed":237,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"5233cd86-cd77-4507-d5ea-c95c46f006db"},"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   document                                             tokens  \\\n","0         7  [Design, Thinking, for, innovation, reflexion,...   \n","1        10  [Diego, Estrada, \\n\\n, Design, Thinking, Assig...   \n","2        16  [Reporting, process, \\n\\n, by, Gilberto, Gambo...   \n","3        20  [Design, Thinking, for, Innovation, \\n\\n, Sind...   \n","4        56  [Assignment, :,   , Visualization,  , Reflecti...   \n","5        86  [Cheese, Startup, -, Learning, Launch, ​by, El...   \n","6        93  [Silvia, Villalobos, \\n\\n, Challenge, :, \\n\\n,...   \n","7       104  [Storytelling,  , The, Path, to, Innovation, \\...   \n","8       112  [Reflection, –, Learning, Launch, \\n\\n, Franci...   \n","9       123  [Gandhi, Institute, of, Technology, and, Manag...   \n","\n","   number of tokens  \n","0               753  \n","1               563  \n","2               729  \n","3              1071  \n","4              1927  \n","5               442  \n","6               225  \n","7               566  \n","8               535  \n","9              1694  "],"text/html":["\n","  <div id=\"df-5123217a-5fcb-457a-a92f-2c41edd9be77\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>document</th>\n","      <th>tokens</th>\n","      <th>number of tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>7</td>\n","      <td>[Design, Thinking, for, innovation, reflexion,...</td>\n","      <td>753</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>10</td>\n","      <td>[Diego, Estrada, \\n\\n, Design, Thinking, Assig...</td>\n","      <td>563</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>16</td>\n","      <td>[Reporting, process, \\n\\n, by, Gilberto, Gambo...</td>\n","      <td>729</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>20</td>\n","      <td>[Design, Thinking, for, Innovation, \\n\\n, Sind...</td>\n","      <td>1071</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>56</td>\n","      <td>[Assignment, :,   , Visualization,  , Reflecti...</td>\n","      <td>1927</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>86</td>\n","      <td>[Cheese, Startup, -, Learning, Launch, ​by, El...</td>\n","      <td>442</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>93</td>\n","      <td>[Silvia, Villalobos, \\n\\n, Challenge, :, \\n\\n,...</td>\n","      <td>225</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>104</td>\n","      <td>[Storytelling,  , The, Path, to, Innovation, \\...</td>\n","      <td>566</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>112</td>\n","      <td>[Reflection, –, Learning, Launch, \\n\\n, Franci...</td>\n","      <td>535</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>123</td>\n","      <td>[Gandhi, Institute, of, Technology, and, Manag...</td>\n","      <td>1694</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5123217a-5fcb-457a-a92f-2c41edd9be77')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-5123217a-5fcb-457a-a92f-2c41edd9be77 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-5123217a-5fcb-457a-a92f-2c41edd9be77');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-62d1f7b9-ba01-4827-a054-5b784eb83817\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-62d1f7b9-ba01-4827-a054-5b784eb83817')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-62d1f7b9-ba01-4827-a054-5b784eb83817 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"train_df","summary":"{\n  \"name\": \"train_df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"document\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 46,\n        \"min\": 7,\n        \"max\": 123,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          112,\n          10,\n          86\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tokens\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"number of tokens\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 554,\n        \"min\": 225,\n        \"max\": 1927,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          535,\n          563,\n          442\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["token_list = train_df.tokens[0]\n","labels_list = train_df.labels[0]\n","row = train_df.iloc[0]\n","\n","def spilt_long_document(row, max_len=400):\n","    doc_number = row.document\n","    doc_tokens = row.tokens\n","    doc_labels = row.labels\n","    new_tokens_labels = []\n","    i = 0\n","    while len(doc_tokens) > max_len:\n","        # check the last token is not a NER tag\n","        if doc_labels[max_len] != 'O':\n","            # print(doc_labels[max_len], doc_labels[max_len+1])\n","            new_max_len = max_len\n","            # itterate through the token list until there is an 'O' and make the split there\n","            while doc_labels[new_max_len] != 'O':\n","                new_max_len += 1\n","            new_tokens = doc_tokens[:new_max_len]\n","            new_labels = doc_labels[:new_max_len]\n","            new_tokens_labels.append({'doc_number': doc_number, 'split': i, 'tokens': new_tokens, 'labels':new_labels})\n","            # update the tokens and labels lists\n","            doc_tokens = doc_tokens[new_max_len:]\n","            doc_labels = doc_labels[new_max_len:]\n","            i += 1\n","\n","        else:\n","            new_tokens = doc_tokens[:max_len]\n","            new_labels = doc_labels[:max_len]\n","            new_tokens_labels.append({'doc_number': doc_number, 'split': i, 'tokens': new_tokens, 'labels':new_labels})\n","            # update the tokens and labels lists\n","            doc_tokens = doc_tokens[max_len:]\n","            doc_labels = doc_labels[max_len:]\n","            i += 1\n","    # write the last part of the list to the new lists\n","    new_tokens_labels.append({'doc_number': doc_number, 'split': i, 'tokens': doc_tokens, 'labels':doc_labels})\n","\n","    # return a dataframe with the new rows (spit cols and labels)\n","    return pd.DataFrame(new_tokens_labels)"],"metadata":{"id":"D7SElo1p_gQy","executionInfo":{"status":"error","timestamp":1714389581626,"user_tz":-60,"elapsed":239,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"colab":{"base_uri":"https://localhost:8080/","height":332},"outputId":"a1696dea-762c-448e-f6b2-a9514f35638d"},"execution_count":4,"outputs":[{"output_type":"error","ename":"AttributeError","evalue":"'DataFrame' object has no attribute 'labels'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)","\u001b[0;32m<ipython-input-4-9f8745252b53>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtoken_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mlabels_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mspilt_long_document\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m400\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5987\u001b[0m         ):\n\u001b[1;32m   5988\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5991\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'labels'"]}]},{"cell_type":"code","source":["# initialize new pandas dataframe\n","new_train_df = pd.DataFrame()\n","\n","# itterate through train_df, split the long texts and insert the new row into the new dataframe\n","for index, row in train_df.iterrows():\n","    # split documents\n","    new_rows = spilt_long_document(row)\n","    # append the new rows to the new_train_df\n","    for i, r in new_rows.iterrows():\n","        new_train_df = pd.concat([new_train_df, pd.DataFrame([new_rows.iloc[i]])], ignore_index=True)"],"metadata":{"id":"GOdnqttpA3oW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["new_train_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xd5VTdUQA6Xc","executionInfo":{"status":"ok","timestamp":1712065002782,"user_tz":-60,"elapsed":231,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"3ea32523-ece0-4b8e-bf28-32fceca579d3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 15889 entries, 0 to 15888\n","Data columns (total 4 columns):\n"," #   Column      Non-Null Count  Dtype \n","---  ------      --------------  ----- \n"," 0   doc_number  15889 non-null  int64 \n"," 1   split       15889 non-null  int64 \n"," 2   tokens      15889 non-null  object\n"," 3   labels      15889 non-null  object\n","dtypes: int64(2), object(2)\n","memory usage: 496.7+ KB\n"]}]},{"cell_type":"code","source":["# create len tokens column\n","new_train_df['number of tokens'] = new_train_df.tokens.apply(len)\n","new_train_df"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":788},"id":"eDELVFnBBuBX","executionInfo":{"status":"ok","timestamp":1712065104844,"user_tz":-60,"elapsed":231,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"f4717ff8-415f-4c02-86c6-d9709729deee"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       doc_number  split                                             tokens  \\\n","0               7      0  [Design, Thinking, for, innovation, reflexion,...   \n","1               7      1  [of, questions, ,, we, can, use, :, who, ,, wh...   \n","2              10      0  [Diego, Estrada, \\n\\n, Design, Thinking, Assig...   \n","3              10      1  [which, eases, the,  , work, of, the, team, .,...   \n","4              16      0  [Reporting, process, \\n\\n, by, Gilberto, Gambo...   \n","...           ...    ...                                                ...   \n","15884       22684      0  [Brainstorming, \\n\\n, Challenge, &, Selection,...   \n","15885       22684      1  [on, which, ones, were, the, best, ., We, came...   \n","15886       22687      0  [Mind, Mapping, \\n\\n, Challenge, \\n\\n, My, con...   \n","15887       22687      1  [had, to, consider, for, their, portions, were...   \n","15888       22687      2  [stating, the, factors, determining,  , wellbe...   \n","\n","                                                  labels  number of tokens  \n","0      [O, O, O, O, O, O, O, O, O, B-NAME_STUDENT, I-...               400  \n","1      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               353  \n","2      [B-NAME_STUDENT, I-NAME_STUDENT, O, O, O, O, O...               400  \n","3      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               163  \n","4      [O, O, O, O, B-NAME_STUDENT, I-NAME_STUDENT, O...               400  \n","...                                                  ...               ...  \n","15884  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               400  \n","15885  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               279  \n","15886  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               400  \n","15887  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...               400  \n","15888  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...                20  \n","\n","[15889 rows x 5 columns]"],"text/html":["\n","  <div id=\"df-e199f2b8-1f6e-4ed1-9a47-8fa70598d3cc\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>doc_number</th>\n","      <th>split</th>\n","      <th>tokens</th>\n","      <th>labels</th>\n","      <th>number of tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>7</td>\n","      <td>0</td>\n","      <td>[Design, Thinking, for, innovation, reflexion,...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, B-NAME_STUDENT, I-...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>7</td>\n","      <td>1</td>\n","      <td>[of, questions, ,, we, can, use, :, who, ,, wh...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>353</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>10</td>\n","      <td>0</td>\n","      <td>[Diego, Estrada, \\n\\n, Design, Thinking, Assig...</td>\n","      <td>[B-NAME_STUDENT, I-NAME_STUDENT, O, O, O, O, O...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>10</td>\n","      <td>1</td>\n","      <td>[which, eases, the,  , work, of, the, team, .,...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>163</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>16</td>\n","      <td>0</td>\n","      <td>[Reporting, process, \\n\\n, by, Gilberto, Gambo...</td>\n","      <td>[O, O, O, O, B-NAME_STUDENT, I-NAME_STUDENT, O...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>15884</th>\n","      <td>22684</td>\n","      <td>0</td>\n","      <td>[Brainstorming, \\n\\n, Challenge, &amp;, Selection,...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>15885</th>\n","      <td>22684</td>\n","      <td>1</td>\n","      <td>[on, which, ones, were, the, best, ., We, came...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>279</td>\n","    </tr>\n","    <tr>\n","      <th>15886</th>\n","      <td>22687</td>\n","      <td>0</td>\n","      <td>[Mind, Mapping, \\n\\n, Challenge, \\n\\n, My, con...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>15887</th>\n","      <td>22687</td>\n","      <td>1</td>\n","      <td>[had, to, consider, for, their, portions, were...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>400</td>\n","    </tr>\n","    <tr>\n","      <th>15888</th>\n","      <td>22687</td>\n","      <td>2</td>\n","      <td>[stating, the, factors, determining,  , wellbe...</td>\n","      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n","      <td>20</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>15889 rows × 5 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e199f2b8-1f6e-4ed1-9a47-8fa70598d3cc')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-e199f2b8-1f6e-4ed1-9a47-8fa70598d3cc button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-e199f2b8-1f6e-4ed1-9a47-8fa70598d3cc');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-0e37a03d-81d2-454b-baf0-7fc7aef6ffd1\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0e37a03d-81d2-454b-baf0-7fc7aef6ffd1')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-0e37a03d-81d2-454b-baf0-7fc7aef6ffd1 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"new_train_df","summary":"{\n  \"name\": \"new_train_df\",\n  \"rows\": 15889,\n  \"fields\": [\n    {\n      \"column\": \"doc_number\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4967,\n        \"min\": 7,\n        \"max\": 22687,\n        \"num_unique_values\": 6807,\n        \"samples\": [\n          17809,\n          11144,\n          16158\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"split\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 8,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          7,\n          1,\n          5\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tokens\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"labels\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"number of tokens\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 124,\n        \"min\": 1,\n        \"max\": 402,\n        \"num_unique_values\": 402,\n        \"samples\": [\n          350,\n          182,\n          258\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["new_train_df.describe()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":300},"id":"_VDeqxR0BuKb","executionInfo":{"status":"ok","timestamp":1712065191040,"user_tz":-60,"elapsed":240,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"c768ad92-c261-4f08-abe6-9fad7471bc91"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["         doc_number         split  number of tokens\n","count  15889.000000  15889.000000      15889.000000\n","mean   14583.960539      0.822896        314.213166\n","std     4967.956870      0.897772        124.220914\n","min        7.000000      0.000000          1.000000\n","25%    10946.000000      0.000000        230.000000\n","50%    14593.000000      1.000000        400.000000\n","75%    18835.000000      1.000000        400.000000\n","max    22687.000000      8.000000        402.000000"],"text/html":["\n","  <div id=\"df-a9bb51c1-3d9e-45c4-95f4-d8d9d53ed915\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>doc_number</th>\n","      <th>split</th>\n","      <th>number of tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>15889.000000</td>\n","      <td>15889.000000</td>\n","      <td>15889.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>14583.960539</td>\n","      <td>0.822896</td>\n","      <td>314.213166</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>4967.956870</td>\n","      <td>0.897772</td>\n","      <td>124.220914</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>7.000000</td>\n","      <td>0.000000</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>10946.000000</td>\n","      <td>0.000000</td>\n","      <td>230.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>14593.000000</td>\n","      <td>1.000000</td>\n","      <td>400.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>18835.000000</td>\n","      <td>1.000000</td>\n","      <td>400.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>22687.000000</td>\n","      <td>8.000000</td>\n","      <td>402.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a9bb51c1-3d9e-45c4-95f4-d8d9d53ed915')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-a9bb51c1-3d9e-45c4-95f4-d8d9d53ed915 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-a9bb51c1-3d9e-45c4-95f4-d8d9d53ed915');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-dfed1929-cfc2-41b4-ae81-3cc406dd8d2c\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-dfed1929-cfc2-41b4-ae81-3cc406dd8d2c')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-dfed1929-cfc2-41b4-ae81-3cc406dd8d2c button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","summary":"{\n  \"name\": \"new_train_df\",\n  \"rows\": 8,\n  \"fields\": [\n    {\n      \"column\": \"doc_number\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 7353.821755845316,\n        \"min\": 7.0,\n        \"max\": 22687.0,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          14583.960538737492,\n          14593.0,\n          15889.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"split\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5617.018448596889,\n        \"min\": 0.0,\n        \"max\": 15889.0,\n        \"num_unique_values\": 6,\n        \"samples\": [\n          15889.0,\n          0.8228963433822141,\n          8.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"number of tokens\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5525.006262571657,\n        \"min\": 1.0,\n        \"max\": 15889.0,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          15889.0,\n          314.2131663414941,\n          400.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","\n","counts = new_train_df['number of tokens']\n","# Plotting the histogram\n","plt.boxplot(counts, showfliers=True)\n","plt.xlabel('Tokens')\n","plt.ylabel('Token Counts')\n","plt.title('Box Plot of Token Counts')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":472},"id":"X_M8Dou8Ckmm","executionInfo":{"status":"ok","timestamp":1712065239836,"user_tz":-60,"elapsed":641,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"63d0a159-fd04-4c58-dd4a-06e20526f8e9"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9AklEQVR4nO3dfXzPdf////t7Y2d2omGbZU6immGGxEI5y8yijqYi1ZzEJw2xcjh2pIU4lE5wlMjnKJNy1KEjOcjZyFmZaJpzfohUs63S9mYYttf3j357fXo3tDfvec/ruF0vl9clr+fz+Xq+Hq/3XOze6+xtMwzDEAAAgEV5uLsAAACAykTYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAaCjR4/KZrMpPT3d3aU4WLlypWJiYuTj4yObzaaCgoJK3Z/NZtOIESMqdR8Arj3CDuBC6enpstlsDktISIi6dOmiFStWXPN61q9f71BL9erVddNNN+mxxx7TN99845J9bN68WRMmTHB5EPn555/14IMPytfXV7NmzdKCBQtUo0aNcuN+/3lfalm/fr1L67tWFi9erPj4eNWuXVteXl4KDw/Xgw8+qM8++8zdpUmScnJyNGHCBGVnZ7u7FOCSqrm7AMCKJk2apEaNGskwDOXl5Sk9PV29evXS0qVLdc8991zzekaNGqW2bdvq/Pnz2r59u+bOnatPP/1Uu3btUnh4+FXNvXnzZk2cOFEDBw5UzZo1XVOwpG3btunkyZN64YUX1L1790uOW7BggcP6u+++q4yMjHLtTZs2dVlt14JhGBo8eLDS09PVqlUrpaSkKCwsTMePH9fixYvVrVs3ffHFF7rjjjvcWmdOTo4mTpyohg0bKiYmxq21AJdC2AEqQXx8vG677TZzfciQIQoNDdU///lPt4SdTp06qW/fvpKkQYMG6ZZbbtGoUaM0f/58paamXvN6KiI/P1+S/jBAPfLIIw7rW7ZsUUZGRrn2682rr76q9PR0jR49Wq+99ppsNpvZ9+yzz2rBggWqVo1/woGK4DIWcA3UrFlTvr6+5X45FRUV6emnn1ZERIS8vb1166236pVXXpFhGJKkM2fOKDIyUpGRkTpz5oy53YkTJ1S3bl3dcccdKikpcbqerl27SpKOHDly2XGfffaZOnXqpBo1aqhmzZq69957tW/fPrN/woQJGjt2rCSpUaNG5iWjo0ePXnbeRYsWqU2bNvL19VXt2rX1yCOP6IcffjD7O3furKSkJElS27ZtZbPZNHDgQKePs8wffc6XM3nyZHl4eOj1118321asWGF+LgEBAUpISNCePXscths4cKD8/f31ww8/6L777pO/v7/q1KmjZ5555g9/ZmfOnNHUqVMVGRmpV155xSHolHn00Ud1++23m+vffPONHnjgAQUHB8vPz0/t27fXp59+6rBN2WXW3/98yi53/vZSX+fOndW8eXPt3btXXbp0kZ+fn2688UZNmzbNYbu2bdtK+jVEl/38y+79OnjwoBITExUWFiYfHx/Vq1dP/fr1U2Fh4WWPH3A1wg5QCQoLC/XTTz/pxx9/1J49ezR8+HCdOnXK4WyDYRjq06ePpk+frp49e+q1117TrbfeqrFjxyolJUWS5Ovrq/nz5+vQoUN69tlnzW2Tk5NVWFio9PR0eXp6Ol3f4cOHJUm1atW65Jg1a9YoLi5O+fn5mjBhglJSUrR582Z16NDB/GV5//33q3///pKk6dOna8GCBVqwYIHq1KlzyXnT09P14IMPytPTU1OnTtXQoUP18ccfq2PHjuZ9P88++6yGDRsm6ddLggsWLND//M//OH2cUsU+50sZP3680tLS9NZbb2nkyJGSfr1slpCQIH9/f7300kt67rnntHfvXnXs2LFciCgpKVFcXJxq1aqlV155RXfddZdeffVVzZ0797L7/fzzz3XixAk9/PDDFfr55uXl6Y477tCqVav05JNPasqUKTp79qz69OmjxYsX/+H2l/LLL7+oZ8+eatmypV599VVFRkZq3Lhx5v1nTZs21aRJkyRJw4YNM3/+d955p86dO6e4uDht2bJFI0eO1KxZszRs2DB98803lX6jOVCOAcBl5s2bZ0gqt3h7exvp6ekOYz/55BNDkjF58mSH9r59+xo2m804dOiQ2Zaammp4eHgYGzduNBYtWmRIMmbMmPGH9axbt86QZLzzzjvGjz/+aOTk5Biffvqp0bBhQ8Nmsxnbtm0zDMMwjhw5Ykgy5s2bZ24bExNjhISEGD///LPZtmPHDsPDw8N47LHHzLaXX37ZkGQcOXLkD+s5d+6cERISYjRv3tw4c+aM2b5s2TJDkpGWlma2lX2WZTVWVHJysvHbf9qc+ZwlGcnJyYZhGMbTTz9teHh4OPzcTp48adSsWdMYOnSow1y5ublGUFCQQ3tSUpIhyZg0aZLD2FatWhlt2rS57DHMnDnTkGQsXry4Qsc8evRoQ5KxadMmh1obNWpkNGzY0CgpKTEM4/8+09//rMr+nqxbt85su+uuuwxJxrvvvmu2FRcXG2FhYUZiYqLZtm3btnJ/dwzDML7++mtDkrFo0aIKHQNQmTizA1SCWbNmKSMjQxkZGXrvvffUpUsXPf744/r444/NMcuXL5enp6dGjRrlsO3TTz8twzAcnt6aMGGCmjVrpqSkJD355JO66667ym13OYMHD1adOnUUHh6uhIQEFRUVaf78+Q73Ff3W8ePHlZ2drYEDByo4ONhsj46O1t13363ly5dXeN+/9dVXXyk/P19PPvmkfHx8zPaEhARFRkaWu+ziCs58ztKvZ4JGjBihmTNn6r333jMvp0lSRkaGCgoK1L9/f/3000/m4unpqXbt2mndunXl9v/EE084rHfq1OkPn4Sz2+2SpICAgAof4+23366OHTuabf7+/ho2bJiOHj2qvXv3Vmie3/P393c4G+nl5aXbb7+9Qk/yBQUFSZJWrVql06dPX9H+AVfh7jagEtx+++0OQaJ///5q1aqVRowYoXvuuUdeXl769ttvFR4eXu4XWtlTQ99++63Z5uXlpXfeeUdt27aVj4+P5s2bd9H7OC4lLS1NnTp1kqenp2rXrq2mTZte9ubWsn3feuut5fqaNm2qVatWqaio6KKPgl/O5eaNjIzU559/7tR8Fd1nRT9n6denuU6dOqXZs2ebl+jKHDx4UNL/3fP0e4GBgQ7rPj4+5S7p3XDDDfrll18uW3PZPCdPnrzsuDLffvut2rVrV679t8fYvHnzCs31W/Xq1Sv39+yGG27Qzp07/3DbRo0aKSUlRa+99pref/99derUSX369NEjjzxiBiHgWuHMDnANeHh4qEuXLjp+/Lj5C9NZq1atkiSdPXvW6TlatGih7t27q0uXLmrRogVP8VxGhw4dFBoaqjfeeEMnTpxw6CstLZX06307ZWfufrssWbLEYfyV3E8l/Rr8JGnXrl1XtP2lXCogX+qG6UvVb1Tgxm7p1yfKdu7cqb/+9a86c+aMRo0apWbNmun777+vWMGAixB2gGvkwoULkqRTp05Jkho0aKCcnJxy//e+f/9+s7/Mzp07NWnSJA0aNEitWrXS448/XqlPtJTt+8CBA+X69u/fr9q1a5tndZw5w3S5eQ8cOOBwzK7izOcsSU2aNNHq1auVk5Ojnj17OmzXuHFjSVJISIi6d+9ebuncubNLau7YsaNuuOEG/fOf/6zQ03YNGjS45M+qrF/69ayMpHI3CP/+7JYz/ujn36JFC40fP14bN27Upk2b9MMPP2jOnDlXvD/gShB2gGvg/PnzWr16tby8vMxLC7169VJJSYneeOMNh7HTp0+XzWZTfHy8ue3AgQMVHh6umTNnKj09XXl5eRozZkyl1Vu3bl3FxMRo/vz5Dr8Yd+/erdWrV6tXr15mW1noqcgTNrfddptCQkI0Z84cFRcXm+0rVqzQvn37lJCQ4LJjKFPRz/m3oqOjtXz5cu3bt0+9e/c2H/uPi4tTYGCg/va3v+n8+fPltvvxxx9dUrOfn5/GjRunffv2ady4cRc9k/Lee+9p69atkn49xq1btyozM9PsLyoq0ty5c9WwYUNFRUVJ+r+wtnHjRnNcSUnJHz4ddjmX+vnb7XYz4Jdp0aKFPDw8HH72wLXAuWygEqxYscL8v+r8/HwtXLhQBw8e1F/+8hfzfozevXurS5cuevbZZ3X06FG1bNlSq1ev1pIlSzR69GjzF9PkyZOVnZ2ttWvXKiAgQNHR0UpLS9P48ePVt29fh+DhSi+//LLi4+MVGxurIUOG6MyZM3r99dcVFBSkCRMmmOPatGkj6dfHxfv166fq1aurd+/eF72fp3r16nrppZc0aNAg3XXXXerfv7/y8vI0c+ZMNWzYsFICXEU/599r3769lixZol69eqlv37765JNPFBgYqNmzZ+vRRx9V69at1a9fP9WpU0fHjh3Tp59+qg4dOpQLVVdq7Nix2rNnj1599VWtW7dOffv2VVhYmHJzc/XJJ59o69at2rx5syTpL3/5i/75z38qPj5eo0aNUnBwsObPn68jR47o3//+tzw8fv3/2mbNmql9+/ZKTU3ViRMnFBwcrA8++KBcKHFG48aNVbNmTc2ZM0cBAQGqUaOG2rVrpx07dmjEiBF64IEHdMstt+jChQtasGCBPD09lZiY6JLPCKgwtz4LBljMxR499/HxMWJiYozZs2cbpaWlDuNPnjxpjBkzxggPDzeqV69u3HzzzcbLL79sjsvKyjKqVatmjBw50mG7CxcuGG3btjXCw8ONX3755ZL1lD1S/EeP/17s0XPDMIw1a9YYHTp0MHx9fY3AwECjd+/ext69e8tt/8ILLxg33nij4eHhUaHH0D/88EOjVatWhre3txEcHGwMGDDA+P777x3GuOrRc8P448+5jH7z6HmZJUuWGNWqVTMeeugh8xHudevWGXFxcUZQUJDh4+NjNG7c2Bg4cKDx1VdfmdslJSUZNWrUKFff888/X66+y/noo4+MHj16GMHBwUa1atWMunXrGg899JCxfv16h3GHDx82+vbta9SsWdPw8fExbr/9dmPZsmXl5jt8+LDRvXt3w9vb2wgNDTX++te/GhkZGRd99LxZs2bltk9KSjIaNGhQ7jOKiooyqlWrZv49+uabb4zBgwcbjRs3Nnx8fIzg4GCjS5cuxpo1ayp87ICr2AyjgneaAQAAXIe4ZwcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaLxXUr993k5OTo4CAAKdefQ8AANzHMAydPHlS4eHh5sszL4awIyknJ0cRERHuLgMAAFyB7777TvXq1btkP2FHUkBAgKRfP6yyV/kDAICqzW63KyIiwvw9fimEHf3ft/YGBgYSdgAAuM780S0o3KAMAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsrcqEnRdffFE2m02jR482286ePavk5GTVqlVL/v7+SkxMVF5ensN2x44dU0JCgvz8/BQSEqKxY8fqwoUL17h6AABQVVWJsLNt2za99dZbio6OdmgfM2aMli5dqkWLFmnDhg3KycnR/fffb/aXlJQoISFB586d0+bNmzV//nylp6crLS3tWh8CAACoomyGYRjuLODUqVNq3bq13nzzTU2ePFkxMTGaMWOGCgsLVadOHS1cuFB9+/aVJO3fv19NmzZVZmam2rdvrxUrVuiee+5RTk6OQkNDJUlz5szRuHHj9OOPP8rLy6tCNdjtdgUFBamwsJBvPQeqgNOnT2v//v0umevMmTM6evSoGjZsKF9f36ueLzIyUn5+fi6oDMDVqujv72rXsKaLSk5OVkJCgrp3767Jkyeb7VlZWTp//ry6d+9utkVGRqp+/fpm2MnMzFSLFi3MoCNJcXFxGj58uPbs2aNWrVpd02MB4Br79+9XmzZt3F3GRWVlZal169buLgOAE9wadj744ANt375d27ZtK9eXm5srLy8v1axZ06E9NDRUubm55pjfBp2y/rK+SykuLlZxcbG5brfbr/QQAJQ5d1rHvl6roqKiq57Ko7hYn/7jby4oSsrJydGbb76pJ598UuHh4Vc9n0feLu377P+76nlq1Kih+q26SV6cJQIqm9vCznfffaennnpKGRkZ8vHxuab7njp1qiZOnHhN9wlY3bGv16r+ikdcNl+My2aSHv8ff6nkXek7F0zmijn+f8f0nuq36+26CQFclNvCTlZWlvLz8x1OB5eUlGjjxo164403tGrVKp07d04FBQUOZ3fy8vIUFhYmSQoLC9PWrVsd5i17WqtszMWkpqYqJSXFXLfb7YqIiHDFYQH/tX621dJ9b53S5MmT1ahRI3eXYyouLlZOTo7Cw8Pl7e3t7nIkSUeOHNH48eP1dq9aqu/uYoD/Am4LO926ddOuXbsc2gYNGqTIyEiNGzdOERERql69utauXavExERJ0oEDB3Ts2DHFxsZKkmJjYzVlyhTl5+crJCREkpSRkaHAwEBFRUVdct/e3t5V5h89wCqMaj76OrdUYa3i1LSK3dMS4+4CfufM9u36OvevMqpd27PawH8rt4WdgIAANW/e3KGtRo0aqlWrltk+ZMgQpaSkKDg4WIGBgRo5cqRiY2PVvn17SVKPHj0UFRWlRx99VNOmTVNubq7Gjx+v5ORkwgwAAJBUBZ7Gupzp06fLw8NDiYmJKi4uVlxcnN58802z39PTU8uWLdPw4cMVGxurGjVqKCkpSZMmTXJj1QAAoCqpUmFn/fr1Dus+Pj6aNWuWZs2adcltGjRooOXLl1dyZQAA4HpVJd6gDAAAUFkIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNLcGnZmz56t6OhoBQYGKjAwULGxsVqxYoXZ37lzZ9lsNofliSeecJjj2LFjSkhIkJ+fn0JCQjR27FhduHDhWh8KAACooqq5c+f16tXTiy++qJtvvlmGYWj+/Pm699579fXXX6tZs2aSpKFDh2rSpEnmNn5+fuafS0pKlJCQoLCwMG3evFnHjx/XY489purVq+tvf/vbNT8eAABQ9bg17PTu3dthfcqUKZo9e7a2bNlihh0/Pz+FhYVddPvVq1dr7969WrNmjUJDQxUTE6MXXnhB48aN04QJE+Tl5VXpxwAAAKq2KnPPTklJiT744AMVFRUpNjbWbH///fdVu3ZtNW/eXKmpqTp9+rTZl5mZqRYtWig0NNRsi4uLk91u1549ey65r+LiYtntdocFAABYk1vP7EjSrl27FBsbq7Nnz8rf31+LFy9WVFSUJOnhhx9WgwYNFB4erp07d2rcuHE6cOCAPv74Y0lSbm6uQ9CRZK7n5uZecp9Tp07VxIkTK+mIAABAVeL2sHPrrbcqOztbhYWF+uijj5SUlKQNGzYoKipKw4YNM8e1aNFCdevWVbdu3XT48GE1btz4iveZmpqqlJQUc91utysiIuKqjgMAAFRNbr+M5eXlpSZNmqhNmzaaOnWqWrZsqZkzZ150bLt27SRJhw4dkiSFhYUpLy/PYUzZ+qXu85Ekb29v8wmwsgUAAFiT28PO75WWlqq4uPiifdnZ2ZKkunXrSpJiY2O1a9cu5efnm2MyMjIUGBhoXgoDAAD/3dx6GSs1NVXx8fGqX7++Tp48qYULF2r9+vVatWqVDh8+rIULF6pXr16qVauWdu7cqTFjxujOO+9UdHS0JKlHjx6KiorSo48+qmnTpik3N1fjx49XcnKyvL293XloAACginBr2MnPz9djjz2m48ePKygoSNHR0Vq1apXuvvtufffdd1qzZo1mzJihoqIiRUREKDExUePHjze39/T01LJlyzR8+HDFxsaqRo0aSkpKcngvDwAA+O/m1rDz9ttvX7IvIiJCGzZs+MM5GjRooOXLl7uyLAAAYCFV7p4dAAAAVyLsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAAS3Nr2Jk9e7aio6MVGBiowMBAxcbGasWKFWb/2bNnlZycrFq1asnf31+JiYnKy8tzmOPYsWNKSEiQn5+fQkJCNHbsWF24cOFaHwoAAKii3Bp26tWrpxdffFFZWVn66quv1LVrV917773as2ePJGnMmDFaunSpFi1apA0bNignJ0f333+/uX1JSYkSEhJ07tw5bd68WfPnz1d6errS0tLcdUgAAKCKqebOnffu3dthfcqUKZo9e7a2bNmievXq6e2339bChQvVtWtXSdK8efPUtGlTbdmyRe3bt9fq1au1d+9erVmzRqGhoYqJidELL7ygcePGacKECfLy8nLHYQEAgCqkytyzU1JSog8++EBFRUWKjY1VVlaWzp8/r+7du5tjIiMjVb9+fWVmZkqSMjMz1aJFC4WGhppj4uLiZLfbzbNDF1NcXCy73e6wAAAAa3J72Nm1a5f8/f3l7e2tJ554QosXL1ZUVJRyc3Pl5eWlmjVrOowPDQ1Vbm6uJCk3N9ch6JT1l/VdytSpUxUUFGQuERERrj0oAABQZbg97Nx6663Kzs7Wl19+qeHDhyspKUl79+6t1H2mpqaqsLDQXL777rtK3R8AAHAft96zI0leXl5q0qSJJKlNmzbatm2bZs6cqYceekjnzp1TQUGBw9mdvLw8hYWFSZLCwsK0detWh/nKntYqG3Mx3t7e8vb2dvGRAACAqsjtZ3Z+r7S0VMXFxWrTpo2qV6+utWvXmn0HDhzQsWPHFBsbK0mKjY3Vrl27lJ+fb47JyMhQYGCgoqKirnntAACg6nHrmZ3U1FTFx8erfv36OnnypBYuXKj169dr1apVCgoK0pAhQ5SSkqLg4GAFBgZq5MiRio2NVfv27SVJPXr0UFRUlB599FFNmzZNubm5Gj9+vJKTkzlzAwAAJLk57OTn5+uxxx7T8ePHFRQUpOjoaK1atUp33323JGn69Ony8PBQYmKiiouLFRcXpzfffNPc3tPTU8uWLdPw4cMVGxurGjVqKCkpSZMmTXLXIQEAgCrGrWHn7bffvmy/j4+PZs2apVmzZl1yTIMGDbR8+XJXlwYAACyiyt2zAwAA4EqEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGlOh53vvvtO33//vbm+detWjR49WnPnznV651OnTlXbtm0VEBCgkJAQ3XfffTpw4IDDmM6dO8tmszksTzzxhMOYY8eOKSEhQX5+fgoJCdHYsWN14cIFp+sBAADW43TYefjhh7Vu3TpJUm5uru6++25t3bpVzz77rCZNmuTUXBs2bFBycrK2bNmijIwMnT9/Xj169FBRUZHDuKFDh+r48ePmMm3aNLOvpKRECQkJOnfunDZv3qz58+crPT1daWlpzh4aAACwoGrObrB7927dfvvtkqR//etfat68ub744gutXr1aTzzxhFMhY+XKlQ7r6enpCgkJUVZWlu68806z3c/PT2FhYRedY/Xq1dq7d6/WrFmj0NBQxcTE6IUXXtC4ceM0YcIEeXl5OXuIAADAQpw+s3P+/Hl5e3tLktasWaM+ffpIkiIjI3X8+PGrKqawsFCSFBwc7ND+/vvvq3bt2mrevLlSU1N1+vRpsy8zM1MtWrRQaGio2RYXFye73a49e/ZcdD/FxcWy2+0OCwAAsCanw06zZs00Z84cbdq0SRkZGerZs6ckKScnR7Vq1briQkpLSzV69Gh16NBBzZs3N9sffvhhvffee1q3bp1SU1O1YMECPfLII2Z/bm6uQ9CRZK7n5uZedF9Tp05VUFCQuURERFxx3QAAoGpz+jLWSy+9pD/96U96+eWXlZSUpJYtW0qS/vOf/5iXt65EcnKydu/erc8//9yhfdiwYeafW7Roobp166pbt246fPiwGjdufEX7Sk1NVUpKirlut9sJPAAAWJTTYadz58766aefZLfbdcMNN5jtw4YNU40aNa6oiBEjRmjZsmXauHGj6tWrd9mx7dq1kyQdOnRIjRs3VlhYmLZu3eowJi8vT5IueZ+Pt7e3eSkOAABYm9OXsbp27aqTJ086BB3p1/tsHnroIafmMgxDI0aM0OLFi/XZZ5+pUaNGf7hNdna2JKlu3bqSpNjYWO3atUv5+fnmmIyMDAUGBioqKsqpegAAgPU4fWZn/fr1OnfuXLn2s2fPatOmTU7NlZycrIULF2rJkiUKCAgw77EJCgqSr6+vDh8+rIULF6pXr16qVauWdu7cqTFjxujOO+9UdHS0JKlHjx6KiorSo48+qmnTpik3N1fjx49XcnIyZ28AAEDFw87OnTvNP+/du9fh5t+SkhKtXLlSN954o1M7nz17tqRfL4391rx58zRw4EB5eXlpzZo1mjFjhoqKihQREaHExESNHz/eHOvp6ally5Zp+PDhio2NVY0aNZSUlOT0O38AAIA1VTjsxMTEmG8w7tq1a7l+X19fvf76607t3DCMy/ZHRERow4YNfzhPgwYNtHz5cqf2DQAA/jtUOOwcOXJEhmHopptu0tatW1WnTh2zz8vLSyEhIfL09KyUIgEAAK5UhcNOgwYNJP36PhwAAIDrhdM3KEvSwYMHtW7dOuXn55cLP3wnFQAAqEqcDjv/+7//q+HDh6t27doKCwuTzWYz+2w2G2EHAABUKU6HncmTJ2vKlCkaN25cZdQDAADgUk6/VPCXX37RAw88UBm1AAAAuJzTYeeBBx7Q6tWrK6MWAAAAl3P6MlaTJk303HPPacuWLWrRooWqV6/u0D9q1CiXFQcAAHC1nA47c+fOlb+/vzZs2FDuhX82m42wAwAAqhSnw86RI0cqow4A17nTp09LkrZv3+7mShydOXNGR48eVcOGDeXr6+vuciRJ+/btc3cJwH+VK3rPDgD83v79+yVJQ4cOdXMl14+AgAB3lwD8V3A67AwePPiy/e+8884VFwPg+nXfffdJkiIjI+Xn5+feYn5j3759euSRR/Tee++padOm7i7HFBAQoJtvvtndZQD/FZwOO7/88ovD+vnz57V7924VFBRc9AtCAfx3qF27th5//HF3l3FJTZs2VevWrd1dBgA3cDrsLF68uFxbaWmphg8frsaNG7ukKAAAAFdx+j07F53Ew0MpKSmaPn26K6YDAABwGZeEHUk6fPiwLly44KrpAAAAXMLpy1gpKSkO64Zh6Pjx4/r000+VlJTkssIAAABcwemw8/XXXzuse3h4qE6dOnr11Vf/8EktAACAa83psLNu3brKqAMAAKBSXPFLBX/88UcdOHBAknTrrbeqTp06LisKAADAVZy+QbmoqEiDBw9W3bp1deedd+rOO+9UeHi4hgwZYr4uHgAAoKpwOuykpKRow4YNWrp0qQoKClRQUKAlS5Zow4YNevrppyujRgAAgCvm9GWsf//73/roo4/UuXNns61Xr17y9fXVgw8+qNmzZ7uyPgAAgKvi9Jmd06dPKzQ0tFx7SEgIl7EAAECV43TYiY2N1fPPP6+zZ8+abWfOnNHEiRMVGxvr0uIAAACultOXsWbOnKm4uDjVq1dPLVu2lCTt2LFDPj4+WrVqlcsLBAAAuBpOh53mzZvr4MGDev/997V//35JUv/+/TVgwAD5+vq6vEAAAICrcUXv2fHz89PQoUNdXQsAAIDLVfienaysLHXp0kV2u71cX2Fhobp06aIdO3a4tDgAAICrVeGw8+qrr6pr164KDAws1xcUFKS7775bL7/8skuLAwAAuFoVDjtffvml7r333kv29+7dW5s3b3ZJUQAAAK5S4bDzww8/KCAg4JL9/v7+On78uFM7nzp1qtq2bauAgACFhITovvvuM79vq8zZs2eVnJysWrVqyd/fX4mJicrLy3MYc+zYMSUkJMjPz08hISEaO3asLly44FQtAADAmiocdurUqVMuiPzW/v37Vbt2bad2vmHDBiUnJ2vLli3KyMjQ+fPn1aNHDxUVFZljxowZo6VLl2rRokXasGGDcnJydP/995v9JSUlSkhI0Llz57R582bNnz9f6enpSktLc6oWAABgUUYFDRw40OjYseNF+0pLS40OHToYAwcOrOh0F5Wfn29IMjZs2GAYhmEUFBQY1atXNxYtWmSO2bdvnyHJyMzMNAzDMJYvX254eHgYubm55pjZs2cbgYGBRnFxcYX2W1hYaEgyCgsLr6p+AFVPVlaWIcnIyspydykAXKyiv78rfGZn/Pjx2rVrl9q1a6d//etf2rFjh3bs2KEPP/xQ7dq10+7du/Xss89eVfAqLCyUJAUHB0v69Qmw8+fPq3v37uaYyMhI1a9fX5mZmZKkzMxMtWjRwuErLOLi4mS327Vnz56L7qe4uFh2u91hAQAA1lTh9+w0btxYa9as0cCBA9WvXz/ZbDZJkmEYioqKUkZGhpo0aXLFhZSWlmr06NHq0KGDmjdvLknKzc2Vl5eXatas6TA2NDRUubm55pjff1dX2XrZmN+bOnWqJk6ceMW1AgCA64dTLxW87bbbtHv3bmVnZ+vgwYMyDEO33HKLYmJirrqQ5ORk7d69W59//vlVz/VHUlNTlZKSYq7b7XZFRERU+n4BAMC1d0VvUI6JiXFJwCkzYsQILVu2TBs3blS9evXM9rCwMJ07d04FBQUOZ3fy8vIUFhZmjtm6davDfGVPa5WN+T1vb295e3u7rH4AAFB1Of2t565kGIZGjBihxYsX67PPPlOjRo0c+tu0aaPq1atr7dq1ZtuBAwd07Ngx8xvWY2NjtWvXLuXn55tjMjIyFBgYqKioqGtzIAAAoMq6ojM7rpKcnKyFCxdqyZIlCggIMO+xCQoKkq+vr4KCgjRkyBClpKQoODhYgYGBGjlypGJjY9W+fXtJUo8ePRQVFaVHH31U06ZNU25ursaPH6/k5GTO3gAAAPeGndmzZ0uSOnfu7NA+b948DRw4UJI0ffp0eXh4KDExUcXFxYqLi9Obb75pjvX09NSyZcs0fPhwxcbGqkaNGkpKStKkSZOu1WEAAIAqzGYYhuHuItzNbrcrKChIhYWFF/3uLwDXr+3bt6tNmzbKyspS69at3V0OABeq6O/vKzqzU1BQoK1btyo/P1+lpaUOfY899tiVTAkAAFApnA47S5cu1YABA3Tq1CkFBgaa79uRJJvNRtgBAABVitNPYz399NMaPHiwTp06pYKCAv3yyy/mcuLEicqoEQAA4Io5HXZ++OEHjRo1Sn5+fpVRDwAAgEs5HXbi4uL01VdfVUYtAAAALuf0PTsJCQkaO3as9u7dqxYtWqh69eoO/X369HFZcQAAAFfL6bAzdOhQSbroe2xsNptKSkquvioAAAAXcTrs/P5RcwAAgKrsqr4b6+zZs66qAwAAoFI4HXZKSkr0wgsv6MYbb5S/v7+++eYbSdJzzz2nt99+2+UFAgAAXA2nw86UKVOUnp6uadOmycvLy2xv3ry5/vGPf7i0OAAAgKvldNh59913NXfuXA0YMECenp5me8uWLbV//36XFgcAAHC1ruilgk2aNCnXXlpaqvPnz7ukKAAAAFdxOuxERUVp06ZN5do/+ugjtWrVyiVFAQAAuIrTj56npaUpKSlJP/zwg0pLS/Xxxx/rwIEDevfdd7Vs2bLKqBEAAOCKOX1m595779XSpUu1Zs0a1ahRQ2lpadq3b5+WLl2qu+++uzJqBAAAuGJOn9n5/vvv1alTJ2VkZJTr27Jli9q3b++SwgAAAFzB6TM7PXr00IkTJ8q1f/HFF+rZs6dLigIAAHAVp8NO+/bt1aNHD508edJs27hxo3r16qXnn3/epcUBAABcLafDzj/+8Q/Vr19fvXv3VnFxsdatW6eEhARNmjRJY8aMqYwaAQAArpjTYcfDw0MffPCBqlevrq5du6pPnz6aOnWqnnrqqcqoDwAA4KpU6AblnTt3lmubMGGC+vfvr0ceeUR33nmnOSY6Otq1FQIAAFyFCoWdmJgY2Ww2GYZhtpWtv/XWW5o7d64Mw5DNZlNJSUmlFQsAAOCsCoWdI0eOVHYdAAAAlaJCYadBgwaVXQcAAEClcPqlgpJ0+PBhzZgxQ/v27ZP06/dlPfXUU2rcuLFLiwMAALhaTj+NtWrVKkVFRWnr1q2Kjo5WdHS0vvzySzVr1uyib1UGAABwJ6fP7PzlL3/RmDFj9OKLL5ZrHzduHN+PBQAAqhSnz+zs27dPQ4YMKdc+ePBg7d271yVFAQAAuIrTYadOnTrKzs4u156dna2QkBBX1AQAAOAyFb6MNWnSJD3zzDMaOnSohg0bpm+++UZ33HGHpF+/BPSll15SSkpKpRUKAABwJSp8ZmfixIk6deqUnnvuOaWlpen111/XXXfdpbvuuktvvPGGJkyYoPHjxzu1840bN6p3794KDw+XzWbTJ5984tA/cOBA2Ww2h+X336x+4sQJDRgwQIGBgapZs6aGDBmiU6dOOVUHAACwrgqf2Sl7e7LNZtOYMWM0ZswY85vPAwICrmjnRUVFatmypQYPHqz777//omN69uypefPmmeve3t4O/QMGDNDx48eVkZGh8+fPa9CgQRo2bJgWLlx4RTUBAABrceppLJvN5rB+pSGnTHx8vOLj4y87xtvbW2FhYRft27dvn1auXKlt27bptttukyS9/vrr6tWrl1555RWFh4dfVX0AAOD651TYueWWW8oFnt87ceLEVRX0e+vXr1dISIhuuOEGde3aVZMnT1atWrUkSZmZmapZs6YZdCSpe/fu8vDw0Jdffqk//elPF52zuLhYxcXF5rrdbndpzQAAoOpwKuxMnDhRQUFBlVVLOT179tT999+vRo0a6fDhw/rrX/+q+Ph4ZWZmytPTU7m5ueWeAKtWrZqCg4OVm5t7yXmnTp2qiRMnVnb5AACgCnAq7PTr1++aPl7er18/888tWrRQdHS0GjdurPXr16tbt25XPG9qaqrDk2N2u10RERFXVSsAAKiaKvw01h9dvroWbrrpJtWuXVuHDh2SJIWFhSk/P99hzIULF3TixIlL3ucj/XofUGBgoMMCAACsqcJhp+xpLHf6/vvv9fPPP6tu3bqSpNjYWBUUFCgrK8sc89lnn6m0tFTt2rVzV5kAAKAKqfBlrNLSUpfv/NSpU+ZZGkk6cuSIsrOzFRwcrODgYE2cOFGJiYkKCwvT4cOH9ec//1lNmjRRXFycJKlp06bq2bOnhg4dqjlz5uj8+fMaMWKE+vXrx5NYAABA0hV8XYQrffXVV2rVqpVatWolSUpJSVGrVq2UlpYmT09P7dy5U3369NEtt9yiIUOGqE2bNtq0aZPDu3bef/99RUZGqlu3burVq5c6duyouXPnuuuQAABAFeP0t567UufOnS97eWzVqlV/OEdwcDAvEAQAAJfk1jM7AAAAlY2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALM2tYWfjxo3q3bu3wsPDZbPZ9Mknnzj0G4ahtLQ01a1bV76+vurevbsOHjzoMObEiRMaMGCAAgMDVbNmTQ0ZMkSnTp26hkcBAACqMreGnaKiIrVs2VKzZs26aP+0adP097//XXPmzNGXX36pGjVqKC4uTmfPnjXHDBgwQHv27FFGRoaWLVumjRs3atiwYdfqEAAAQBVXzZ07j4+PV3x8/EX7DMPQjBkzNH78eN17772SpHfffVehoaH65JNP1K9fP+3bt08rV67Utm3bdNttt0mSXn/9dfXq1UuvvPKKwsPDr9mxAACAqqnK3rNz5MgR5ebmqnv37mZbUFCQ2rVrp8zMTElSZmamatasaQYdSerevbs8PDz05ZdfXnLu4uJi2e12hwUAAFhTlQ07ubm5kqTQ0FCH9tDQULMvNzdXISEhDv3VqlVTcHCwOeZipk6dqqCgIHOJiIhwcfUAAKCqqLJhpzKlpqaqsLDQXL777jt3lwQAACpJlQ07YWFhkqS8vDyH9ry8PLMvLCxM+fn5Dv0XLlzQiRMnzDEX4+3trcDAQIcFAABYU5UNO40aNVJYWJjWrl1rttntdn355ZeKjY2VJMXGxqqgoEBZWVnmmM8++0ylpaVq167dNa8ZAABUPW59GuvUqVM6dOiQuX7kyBFlZ2crODhY9evX1+jRozV58mTdfPPNatSokZ577jmFh4frvvvukyQ1bdpUPXv21NChQzVnzhydP39eI0aMUL9+/XgSCwAASHJz2Pnqq6/UpUsXcz0lJUWSlJSUpPT0dP35z39WUVGRhg0bpoKCAnXs2FErV66Uj4+Puc3777+vESNGqFu3bvLw8FBiYqL+/ve/X/NjAQAAVZPNMAzD3UW4m91uV1BQkAoLC7l/B7CY7du3q02bNsrKylLr1q3dXQ4AF6ro7+8qe88OAACAKxB2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApVXpsDNhwgTZbDaHJTIy0uw/e/askpOTVatWLfn7+ysxMVF5eXlurBgAAFQ1VTrsSFKzZs10/Phxc/n888/NvjFjxmjp0qVatGiRNmzYoJycHN1///1urBYAAFQ11dxdwB+pVq2awsLCyrUXFhbq7bff1sKFC9W1a1dJ0rx589S0aVNt2bJF7du3v9alAgCAKqjKn9k5ePCgwsPDddNNN2nAgAE6duyYJCkrK0vnz59X9+7dzbGRkZGqX7++MjMzLztncXGx7Ha7wwIAAKypSoeddu3aKT09XStXrtTs2bN15MgRderUSSdPnlRubq68vLxUs2ZNh21CQ0OVm5t72XmnTp2qoKAgc4mIiKjEowAAAO5UpS9jxcfHm3+Ojo5Wu3bt1KBBA/3rX/+Sr6/vFc+bmpqqlJQUc91utxN4AACwqCp9Zuf3atasqVtuuUWHDh1SWFiYzp07p4KCAocxeXl5F73H57e8vb0VGBjosAAAAGu6rsLOqVOndPjwYdWtW1dt2rRR9erVtXbtWrP/wIEDOnbsmGJjY91YJQAAqEqq9GWsZ555Rr1791aDBg2Uk5Oj559/Xp6enurfv7+CgoI0ZMgQpaSkKDg4WIGBgRo5cqRiY2N5EgsAAJiqdNj5/vvv1b9/f/3888+qU6eOOnbsqC1btqhOnTqSpOnTp8vDw0OJiYkqLi5WXFyc3nzzTTdXDQAAqpIqHXY++OCDy/b7+Pho1qxZmjVr1jWqCAAAXG+uq3t2AAAAnEXYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlmaZsDNr1iw1bNhQPj4+ateunbZu3erukgAAQBVgibDz4YcfKiUlRc8//7y2b9+uli1bKi4uTvn5+e4uDQAAuJklws5rr72moUOHatCgQYqKitKcOXPk5+end955x92lAQAAN6vm7gKu1rlz55SVlaXU1FSzzcPDQ927d1dmZqYbKwNwpU6fPq39+/e7ZK59+/Y5/PdqRUZGys/PzyVzAbg2rvuw89NPP6mkpEShoaEO7aGhoZf8x7K4uFjFxcXmut1ur9QaAThn//79atOmjUvnfOSRR1wyT1ZWllq3bu2SuQBcG9d92LkSU6dO1cSJE91dBoBLiIyMVFZWlkvmOnPmjI4ePaqGDRvK19f3queLjIx0QVUArqXrPuzUrl1bnp6eysvLc2jPy8tTWFjYRbdJTU1VSkqKuW632xUREVGpdQKoOD8/P5eePenQoYPL5gJw/bnub1D28vJSmzZttHbtWrOttLRUa9euVWxs7EW38fb2VmBgoMMCAACs6bo/syNJKSkpSkpK0m233abbb79dM2bMUFFRkQYNGuTu0gAAgJtZIuw89NBD+vHHH5WWlqbc3FzFxMRo5cqV5W5aBgAA/31shmEY7i7C3ex2u4KCglRYWMglLQAArhMV/f193d+zAwAAcDmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmW+LqIq1X2Emm73e7mSgAAQEWV/d7+oy+DIOxIOnnypCQpIiLCzZUAAABnnTx5UkFBQZfs57uxJJWWlionJ0cBAQGy2WzuLgeAC9ntdkVEROi7777ju+8AizEMQydPnlR4eLg8PC59Zw5hB4Cl8UW/ALhBGQAAWBphBwAAWBphB4CleXt76/nnn5e3t7e7SwHgJtyzAwAALI0zOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAsaePGjerdu7fCw8Nls9n0ySefuLskAG5C2AFgSUVFRWrZsqVmzZrl7lIAuBlfBArAkuLj4xUfH+/uMgBUAZzZAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlsbTWAAs6dSpUzp06JC5fuTIEWVnZys4OFj169d3Y2UArjW+9RyAJa1fv15dunQp156UlKT09PRrXxAAtyHsAAAAS+OeHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQDXtaNHj8pmsyk7O9vdpQCoogg7ANzOZrNddpkwYYK7SwRwHeO7sQC43fHjx80/f/jhh0pLS9OBAwfMNn9/f3eUBcAiOLMDwO3CwsLMJSgoSDabzVwPCQnRa6+9pnr16snb21sxMTFauXLlJecqKSnR4MGDFRkZqWPHjkmSlixZotatW8vHx0c33XSTJk6cqAsXLpjb2Gw2/eMf/9Cf/vQn+fn56eabb9Z//vMfs/+XX37RgAEDVKdOHfn6+urmm2/WvHnzKu8DAeBShB0AVdrMmTP16quv6pVXXtHOnTsVFxenPn366ODBg+XGFhcX64EHHlB2drY2bdqk+vXra9OmTXrsscf01FNPae/evXrrrbeUnp6uKVOmOGw7ceJEPfjgg9q5c6d69eqlAQMG6MSJE5Kk5557Tnv37tWKFSu0b98+zZ49W7Vr174mxw/ABQwAqELmzZtnBAUFmevh4eHGlClTHMa0bdvWePLJJw3DMIwjR44YkoxNmzYZ3bp1Mzp27GgUFBSYY7t162b87W9/c9h+wYIFRt26dc11Scb48ePN9VOnThmSjBUrVhiGYRi9e/c2Bg0a5LJjBHBtcc8OgCrLbrcrJydHHTp0cGjv0KGDduzY4dDWv39/1atXT5999pl8fX3N9h07duiLL75wOJNTUlKis2fP6vTp0/Lz85MkRUdHm/01atRQYGCg8vPzJUnDhw9XYmKitm/frh49eui+++7THXfc4fLjBVA5uIwFwBJ69eqlnTt3KjMz06H91KlTmjhxorKzs81l165dOnjwoHx8fMxx1atXd9jOZrOptLRUkhQfH69vv/1WY8aMUU5Ojrp166Znnnmm8g8KgEsQdgBUWYGBgQoPD9cXX3zh0P7FF18oKirKoW348OF68cUX1adPH23YsMFsb926tQ4cOKAmTZqUWzw8Kv5PYJ06dZSUlKT33ntPM2bM0Ny5c6/u4ABcM1zGAlCljR07Vs8//7waN26smJgYzZs3T9nZ2Xr//ffLjR05cqRKSkp0zz33aMWKFerYsaPS0tJ0zz33qH79+urbt688PDy0Y8cO7d69W5MnT65QDWlpaWrTpo2aNWum4uJiLVu2TE2bNnX1oQKoJIQdAFXaqFGjVFhYqKefflr5+fmKiorSf/7zH918880XHT969GiVlpaqV69eWrlypeLi4rRs2TJNmjRJL730kqpXr67IyEg9/vjjFa7By8tLqampOnr0qHx9fdWpUyd98MEHrjpEAJXMZhiG4e4iAAAAKgv37AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEv7f3FoXZmtkULVAAAAAElFTkSuQmCC\n"},"metadata":{}}]},{"cell_type":"code","source":["# save new dataset as csv\n","new_train_df.to_csv('/content/drive/MyDrive/Colab Notebooks/Kaggle competition /train_short.csv', index=False)\n","# save as json\n","new_train_df.to_json('/content/drive/MyDrive/Colab Notebooks/Kaggle competition /train_short.json', orient='records', lines=True)"],"metadata":{"id":"CsmL4Xb9Cmmg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_exploded = new_train_df.explode('labels')\n","# Count the occurrences of each unique string\n","label_counts = df_exploded['labels'].value_counts()\n","\n","# Display the counts\n","print(label_counts)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3NCDuKblH7Hc","executionInfo":{"status":"ok","timestamp":1712066643666,"user_tz":-60,"elapsed":2073,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"03a605d5-d3f8-4543-a299-19e874c45846"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["O                   4989794\n","B-NAME_STUDENT         1365\n","I-NAME_STUDENT         1096\n","B-URL_PERSONAL          110\n","B-ID_NUM                 78\n","B-EMAIL                  39\n","I-STREET_ADDRESS         20\n","I-PHONE_NUM              15\n","B-USERNAME                6\n","B-PHONE_NUM               6\n","B-STREET_ADDRESS          2\n","I-URL_PERSONAL            1\n","I-ID_NUM                  1\n","Name: labels, dtype: int64\n"]}]},{"cell_type":"code","source":["label_list = list(label_counts.index)\n","label_list"],"metadata":{"id":"ErU0IlP3IIUM","executionInfo":{"status":"ok","timestamp":1712066697827,"user_tz":-60,"elapsed":300,"user":{"displayName":"Mihan Perera","userId":"17729192883454716790"}},"outputId":"64f99fcb-6819-4d4b-db22-c455c802c494","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['O',\n"," 'B-NAME_STUDENT',\n"," 'I-NAME_STUDENT',\n"," 'B-URL_PERSONAL',\n"," 'B-ID_NUM',\n"," 'B-EMAIL',\n"," 'I-STREET_ADDRESS',\n"," 'I-PHONE_NUM',\n"," 'B-USERNAME',\n"," 'B-PHONE_NUM',\n"," 'B-STREET_ADDRESS',\n"," 'I-URL_PERSONAL',\n"," 'I-ID_NUM']"]},"metadata":{},"execution_count":12}]}]}