User Rep Score - multiple files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to add an question base label column to each row in answers_df\n",
    "\n",
    "def get_question_base_label(row):\n",
    "    if \"A\" not in row['Question Label']:\n",
    "        return row['Question Label']\n",
    "    else:\n",
    "        base_end_index = row['Question Label'].index(\"A\") - 1\n",
    "        return row['Question Label'][:base_end_index]\n",
    "    \n",
    "\n",
    "# functions for each question type that take in a row from parent_df and output a \n",
    "# row with the relevant information (including score) for that user.\n",
    "\n",
    "def ordinal_question(row, task_answers_df, num_answers, parent_questions):\n",
    "    article_number = row[\"article_number\"]\n",
    "    question_label = row[\"question_label\"]\n",
    "    answer_label = int(row[\"answer_label\"][-1])\n",
    "    correct_answer = task_answers_df.loc[task_answers_df[\"Article Number\"] == article_number]\n",
    "    correct_answer = correct_answer.loc[correct_answer[\"Question Base Label\"] == question_label]\n",
    "    correct_answer = int(correct_answer[\"Answer Label\"].iloc[0][-1])\n",
    "    \n",
    "    question_num_answers = num_answers[parent_questions.index(row[\"question_label\"])]\n",
    "    penalty = 1 / (question_num_answers / 2)\n",
    "    \n",
    "    return 1 - abs(answer_label - correct_answer) * penalty\n",
    "\n",
    "def select_one_question(row, task_answers_df):\n",
    "    article_number = row[\"article_number\"]\n",
    "    question_label = row[\"question_label\"]\n",
    "    answer_label = row[\"answer_label\"]\n",
    "    correct_answer = task_answers_df.loc[task_answers_df[\"Article Number\"] == article_number]\n",
    "    correct_answer = correct_answer.loc[correct_answer[\"Question Base Label\"] == question_label]\n",
    "    correct_answer = correct_answer[\"Answer Label\"].iloc[0]\n",
    "    \n",
    "    if answer_label == correct_answer:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def select_all_question(row, task_answers_df, parent_df):\n",
    "    user_id = row[\"contributor_uuid\"]\n",
    "    response_id = row[\"quiz_taskrun_uuid\"]\n",
    "    \n",
    "    if user_dfs[user_id].loc[user_dfs[user_id][\"quiz_taskrun_uuid\"] == response_id].empty:\n",
    "        article_number = row[\"article_number\"]\n",
    "        question_label = row[\"question_label\"]\n",
    "        answer_label = row[\"answer_label\"]\n",
    "\n",
    "        question_user_df = parent_df.loc[parent_df[\"quiz_taskrun_uuid\"] == response_id]\n",
    "        question_answers_df = task_answers_df.loc[(task_answers_df[\"Article Number\"] == article_number) &\\\n",
    "                                                 (task_answers_df[\"Question Base Label\"] == question_label)]\n",
    "\n",
    "        correct_question_counter = 0\n",
    "        total_question_counter = 0\n",
    "        \n",
    "        for index, row in question_answers_df.iterrows():\n",
    "            if question_user_df.loc[question_user_df[\"answer_label\"] == row[\"Question Label\"]].empty:\n",
    "                if row[\"Answer Label\"] == \"0\":\n",
    "                    correct_question_counter += 1\n",
    "            else:\n",
    "                if row[\"Answer Label\"] == \"1\":\n",
    "                    correct_question_counter += 1\n",
    "            total_question_counter += 1\n",
    "\n",
    "        overall_question_score = correct_question_counter / total_question_counter\n",
    "\n",
    "        return overall_question_score\n",
    "    \n",
    "    return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup\n",
    "answers_df = pd.read_csv(\"convergence/Answer_Consensus.csv\")\n",
    "\n",
    "# array of questions that are \"parent\" questions\n",
    "language_parents = [\"T1.Q1\", \"T1.Q12\"]\n",
    "probability_parents = [\"T1.Q1\", \"T1.Q5\", \"T1.Q6\", \"T1.Q11\"]\n",
    "reasoning_parents = [\"T1.Q1\"]\n",
    "evidence_parents = [\"T1.Q1\", \"T1.Q12\"]\n",
    "\n",
    "# corresponding list of question types, language_parents[n] maps to language_parent_types[n]\n",
    "language_parents_types = [\"select_all\", \"ordinal\"]\n",
    "probability_parents_types = [\"ordinal\", \"ordinal\", \"select_one\", \"ordinal\"]\n",
    "reasoning_parents_types = [\"select_all\"]\n",
    "evidence_parents_types = [\"select_one\", \"ordinal\"]\n",
    "\n",
    "# corresponding list of max number of problems, used for scoring ordinal questions\n",
    "language_num_answers = [13, 4]\n",
    "probability_num_answers = [3, 3, 3, 4]\n",
    "reasoning_num_answers = [6]\n",
    "evidence_num_answers = [3, 4]\n",
    "\n",
    "user_dfs = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user rep score for single file\n",
    "def single_file(datahunt_file):\n",
    "    df = pd.read_csv(datahunt_file)\n",
    "    \n",
    "    file_answers_df = answers_df.copy()\n",
    "    \n",
    "    # assigns a task type based on the name of the inputted datahunt csv\n",
    "    if \"language\" in datahunt_file.lower():\n",
    "        task_type = \"language\"\n",
    "    elif \"probability\" in datahunt_file.lower():\n",
    "        task_type = \"probability\"\n",
    "    elif \"reasoning\" in datahunt_file.lower():\n",
    "        task_type = \"reasoning\"\n",
    "    else:\n",
    "        task_type = \"evidence\"\n",
    "        \n",
    "    file_answers_df[\"Question Base Label\"] = file_answers_df.apply(get_question_base_label, axis=1)\n",
    "    \n",
    "    # cuts down file_answers_df to the rows with the relevant task type\n",
    "    if task_type == \"language\":\n",
    "        task_answers_df = file_answers_df.loc[file_answers_df[\"Task File\"] == \"Language\"]\n",
    "        parent_questions = language_parents\n",
    "        num_answers = language_num_answers\n",
    "    elif task_type == \"probability\":\n",
    "        task_answers_df = file_answers_df.loc[file_answers_df[\"Task File\"] == \"Probability\"]\n",
    "        parent_questions = probability_parents\n",
    "        num_answers = probability_num_answers\n",
    "    elif task_type == \"reasoning\":\n",
    "        task_answers_df = file_answers_df.loc[file_answers_df[\"Task File\"] == \"Reasoning\"]\n",
    "        parent_questions = reasoning_parents\n",
    "        num_answers = reasoning_num_answers\n",
    "    elif task_type == \"evidence\":\n",
    "        task_answers_df = file_answers_df.loc[file_answers_df[\"Task File\"] == \"Evidence\"]\n",
    "        parent_questions = evidence_parents\n",
    "        num_answers = evidence_num_answers\n",
    "    else:\n",
    "        print(\"Invalid task type\")\n",
    "    \n",
    "    # set parent_df to the relevant columns of the parent questions\n",
    "    df = df[[\"contributor_uuid\", \"quiz_task_uuid\", \"article_number\", \"question_label\", \n",
    "             \"answer_label\", \"answer_text\", \"quiz_taskrun_uuid\", \"finish_time\"]]\n",
    "\n",
    "    # creates a dictionary mapping the task type to an list of which questions are parent questions\n",
    "    task_parents = {\"language\": language_parents, \"probability\": probability_parents,\n",
    "                    \"reasoning\": reasoning_parents, \"evidence\": evidence_parents}\n",
    "    task_parents_types = {\"language\": language_parents_types, \"probability\": probability_parents_types,\n",
    "                    \"reasoning\": reasoning_parents_types, \"evidence\": evidence_parents_types}\n",
    "\n",
    "\n",
    "    parent_df = df.loc[df['question_label'].isin(task_parents[task_type])]\n",
    "    \n",
    "    # adds a column with the question_type of the question_label\n",
    "    question_types = []\n",
    "    for i in parent_df['question_label']:\n",
    "        question_type = task_parents_types[task_type][task_parents[task_type].index(i)]\n",
    "        question_types.append(question_type)\n",
    "\n",
    "    parent_df.insert(2, \"question_type\", question_types, True)\n",
    "    \n",
    "\n",
    "    for index, row in parent_df.iterrows():\n",
    "        if row['contributor_uuid'] not in user_dfs:\n",
    "            user_dfs[row['contributor_uuid']] = pd.DataFrame(columns=['quiz_task_uuid', 'quiz_taskrun_uuid', 'question_label', 'question_type', 'answer_score', 'time_stamp'])\n",
    "        if row['question_type'] == \"select_all\":\n",
    "            score = select_all_question(row, task_answers_df, parent_df)\n",
    "        elif row['question_type'] == \"select_one\":\n",
    "            score = select_one_question(row, task_answers_df)\n",
    "        else:\n",
    "            score = ordinal_question(row, task_answers_df, num_answers, parent_questions)\n",
    "        if score != -1:\n",
    "            user_dfs[row['contributor_uuid']] = user_dfs[row['contributor_uuid']].append({'quiz_task_uuid': row['quiz_task_uuid'], 'quiz_taskrun_uuid': row['quiz_taskrun_uuid'], 'question_label': row['question_label'], 'task_type': task_type, 'question_type': row['question_type'], 'answer_score': score, 'time_stamp': row['finish_time']}, ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def task_rep_score(user_df, task_type):\n",
    "    tt_user_df = user_df.loc[user_df[\"task_type\"] == task_type]\n",
    "\n",
    "    if(len(tt_user_df) == 0):\n",
    "        return 0\n",
    "    \n",
    "    else:\n",
    "        temp = tt_user_df.time_stamp.unique()\n",
    "        temp[::-1].sort()\n",
    "\n",
    "        temp = temp[:30]\n",
    "\n",
    "        return tt_user_df.loc[tt_user_df[\"time_stamp\"].isin(temp)].answer_score.mean()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = r'testing-format'\n",
    "for filename in os.listdir(directory):\n",
    "    if filename.endswith(\".csv\"):\n",
    "        single_file(os.path.join(directory, filename))\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "user_rep_scores = {}\n",
    "\n",
    "for user in user_dfs:\n",
    "    l_urs = task_rep_score(user_dfs[user], \"language\")\n",
    "    p_urs = task_rep_score(user_dfs[user], \"probability\")\n",
    "    r_urs = task_rep_score(user_dfs[user], \"reasoning\")\n",
    "    e_urs = task_rep_score(user_dfs[user], \"evidence\")\n",
    "    total_urs = np.mean([l_urs, p_urs, r_urs, e_urs])\n",
    "    \n",
    "    data = {'task-type':  ['Language', 'Probability', 'Reasoning', 'Evidence', 'Total User Rep Score'],\n",
    "        'rep-score': [l_urs, p_urs, r_urs, e_urs, total_urs]}\n",
    "    \n",
    "    urs_df = pd.DataFrame (data, columns = ['task-type', 'rep-score'])\n",
    "    \n",
    "    urs_df.to_csv(\"userRepScore-csvs/\" + user)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}