ReviewComments/PR_review_comments_stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "from collections import defaultdict\n",
    "import csv\n",
    "import lzma\n",
    "import operator\n",
    "import re\n",
    "from statistics import mean\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_CSV = \"/home/waren/sourced/data/datasets/review_comments.csv.xz\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getusers(filename):\n",
    "    with lzma.open(filename) as archf:\n",
    "        reader = csv.DictReader(codecs.getreader(\"utf-8\")(archf))\n",
    "        for record in reader:\n",
    "            yield record[\"AUTHOR\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25300000\r"
     ]
    }
   ],
   "source": [
    "n_comments = 0\n",
    "users = defaultdict(int)\n",
    "for user in getusers(PATH_TO_CSV):\n",
    "    n_comments += 1\n",
    "    if n_comments % 100000 == 0:\n",
    "        sys.stderr.write(\"%d\\r\" % n_comments)\n",
    "    users[user] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of PR review comments: 25323640\n",
      "Number of different reviewers: 540054\n",
      "Average number of GitHub review comments per user from 2015 to 2019: 46\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of PR review comments: %d\" % (n_comments))\n",
    "print(\"Number of different reviewers: %d\" % (len(users)))\n",
    "print(\"Average number of GitHub review comments per user from 2015 to 2019: %d\" % (mean(list(users.values()))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Most active reviewers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('houndci-bot', 797827),\n",
       " ('houndci', 264998),\n",
       " ('codacy-bot', 237814),\n",
       " ('stickler-ci', 36707),\n",
       " ('sonarcloud[bot]', 20378),\n",
       " ('jreback', 19694),\n",
       " ('seanlip', 19540),\n",
       " ('codeschool-kiddo', 18946),\n",
       " ('stephentoub', 18744),\n",
       " ('vkurennov', 18141)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_users = sorted(users.items(), key=operator.itemgetter(1), reverse=True)\n",
    "sorted_users[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of review comments about typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "typos = r\"(?i)\\b(typo)\\b\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "113200\r"
     ]
    }
   ],
   "source": [
    "c = 0\n",
    "typos_comments = []\n",
    "with lzma.open(PATH_TO_CSV) as archf:\n",
    "    reader = csv.DictReader(codecs.getreader(\"utf-8\")(archf))\n",
    "    for record in reader:\n",
    "        if c % 100 == 0:\n",
    "            sys.stderr.write(\"%d\\r\" % c)\n",
    "        try:\n",
    "            if re.search(typos, record[\"BODY\"]):\n",
    "                c += 1\n",
    "                typos_comments.append(record[\"BODY\"])\n",
    "        except TypeError:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of PR review comments about typos: 113272\n",
      "Overall proportion amount all review comments: 0.45%\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of PR review comments about typos: %d\" % (len(typos_comments)))\n",
    "print(\"Overall proportion amount all review comments: %.2f%%\" % (100 * len(typos_comments) / n_comments))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['There is a slight typo here. I will point it out to you in class',\n",
       " '@Nylanfs \\r\\nUh... Summoner slayer syes? Is this intentional or a typo?',\n",
       " \"It's a better practice to put constant first in a comparison. Consider this comparison typo issue:\\r\\n``` javascript\\r\\nif (attribute = 'aValue')\\r\\n```\\r\\n``` javascript\\r\\nif ('aValue' = attribute)\\r\\n```\\r\\nThe second form is really easier to detect\\r\\n\\r\\nBut I can change it if it breaks project coding style\",\n",
       " 'UNI_TIMESTAMP is that a typo ?',\n",
       " 'typo ;) \"vestivals\"',\n",
       " 'typo',\n",
       " 'typo!',\n",
       " 'typo!',\n",
       " 'You made a little typo, fixing ;)\\r\\n@eiriktsarpalis',\n",
       " 'typo in the package name (\"performaceplatform\"), is this working?']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "typos_comments[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}