feat: implement grading strategies for peer evaluations

openassessment/__init__.py

@@ -2,4 +2,4 @@
 Initialization Information for Open Assessment Module
 """
 
-__version__ = '6.9.0'
+__version__ = '6.10.0'

openassessment/assessment/api/peer.py

@@ -8,14 +8,15 @@
 
 import logging
 
+from django.conf import settings
 from django.db import DatabaseError, IntegrityError, transaction
 from django.utils import timezone
 
 from submissions import api as sub_api
 from openassessment.assessment.errors import (PeerAssessmentInternalError, PeerAssessmentRequestError,
                                               PeerAssessmentWorkflowError)
 from openassessment.assessment.models import (Assessment, AssessmentFeedback, AssessmentPart, InvalidRubricSelection,
-                                              PeerWorkflow, PeerWorkflowItem)
+                                              PeerWorkflow, PeerWorkflowItem, PeerGradingStrategy)
 from openassessment.assessment.serializers import (AssessmentFeedbackSerializer, InvalidRubric, RubricSerializer,
                                                    full_assessment_dict, rubric_from_dict, serialize_assessments)
 
@@ -51,6 +52,25 @@ def flexible_peer_grading_active(submission_uuid, peer_requirements, course_sett
     return days_elapsed >= FLEXIBLE_PEER_GRADING_REQUIRED_SUBMISSION_AGE_IN_DAYS
 
 
+def get_peer_grading_strategy(workflow_requirements):
+    """
+    Get the peer grading type, either mean or median. If no grading strategy is
+    provided in the peer requirements or the feature flag is not enabled, the
+    default median score calculation is used.
+    """
+    # If the feature flag is not enabled, use the median score calculation
+    # as the default behavior.
+    if not settings.FEATURES.get("ENABLE_ORA_PEER_CONFIGURABLE_GRADING", False):
+        return PeerGradingStrategy.MEDIAN
+
+    if "peer" not in workflow_requirements:
+        return workflow_requirements.get("grading_strategy", PeerGradingStrategy.MEDIAN)
+
+    return workflow_requirements.get("peer", {}).get(
+        "grading_strategy", PeerGradingStrategy.MEDIAN,
+    )
+
+
 def required_peer_grades(submission_uuid, peer_requirements, course_settings):
     """
     Given a submission id, finds how many peer assessment required.
@@ -281,10 +301,12 @@ def get_score(submission_uuid, peer_requirements, course_settings):
         scored_item.save()
     assessments = [item.assessment for item in items]
 
+    scores_dict = get_assessment_scores_with_grading_strategy(
+        submission_uuid,
+        peer_requirements,
+    )
     return {
-        "points_earned": sum(
-            get_assessment_median_scores(submission_uuid).values()
-        ),
+        "points_earned": sum(scores_dict.values()),
         "points_possible": assessments[0].points_possible,
         "contributing_assessments": [assessment.id for assessment in assessments],
         "staff_id": None,
@@ -501,36 +523,37 @@ def get_rubric_max_scores(submission_uuid):
         raise PeerAssessmentInternalError(error_message) from ex
 
 
-def get_assessment_median_scores(submission_uuid):
-    """Get the median score for each rubric criterion
+def get_assessment_scores_with_grading_strategy(submission_uuid, workflow_requirements):
+    """Get the score for each rubric criterion calculated given grading strategy
+    obtained from the peer requirements dictionary.
 
-    For a given assessment, collect the median score for each criterion on the
-    rubric. This set can be used to determine the overall score, as well as each
-    part of the individual rubric scores.
-
-    If there is a true median score, it is returned. If there are two median
-    values, the average of those two values is returned, rounded up to the
-    greatest integer value.
+    This function is based on the archived get_assessment_median_scores, but allows the caller
+    to specify the grading strategy (mean, median) to use when calculating the score.
 
     Args:
         submission_uuid (str): The submission uuid is used to get the
             assessments used to score this submission, and generate the
-            appropriate median score.
+            appropriate median/mean score.
+        workflow_requirements (dict): Dictionary with the key "grading_strategy"
 
     Returns:
         dict: A dictionary of rubric criterion names,
-        with a median score of the peer assessments.
+        with a median/mean score of the peer assessments.
 
     Raises:
         PeerAssessmentInternalError: If any error occurs while retrieving
-            information to form the median scores, an error is raised.
+            information to form the median/mean scores, an error is raised.
     """
+    current_grading_strategy = get_peer_grading_strategy(workflow_requirements)
     try:
         workflow = PeerWorkflow.objects.get(submission_uuid=submission_uuid)
         items = workflow.graded_by.filter(scored=True)
         assessments = [item.assessment for item in items]
         scores = Assessment.scores_by_criterion(assessments)
-        return Assessment.get_median_score_dict(scores)
+        return Assessment.get_score_dict(
+            scores,
+            grading_strategy=current_grading_strategy,
+        )
     except PeerWorkflow.DoesNotExist:
         return {}
     except DatabaseError as ex:

openassessment/assessment/models/base.py

@@ -36,6 +36,12 @@
 KEY_SEPARATOR = '/'
 
 
+class PeerGradingStrategy:
+    """Grading strategies for peer assessments."""
+    MEAN = "mean"
+    MEDIAN = "median"
+
+
 class InvalidRubricSelection(Exception):
     """
     The specified criterion/option do not exist in the rubric.
@@ -488,6 +494,27 @@ def create(cls, rubric, scorer_id, submission_uuid, score_type, feedback=None, s
 
         return cls.objects.create(**assessment_params)
 
+    @classmethod
+    def get_score_dict(cls, scores_dict, grading_strategy):
+        """Determine the score in a dictionary of lists of scores based on the
+        grading strategy calculation configuration.
+
+        Args:
+            scores_dict (dict): A dictionary of lists of int values. These int values
+                are reduced to a single value that represents the median.
+            grading_strategy (str): The type of score to calculate.  Defaults to "median".
+
+        Returns:
+            (dict): A dictionary with criterion name keys and median score
+                values.
+        """
+        assert grading_strategy in [
+            PeerGradingStrategy.MEDIAN,
+            PeerGradingStrategy.MEAN,
+        ], "Invalid grading strategy."
+
+        return getattr(cls, f"get_{grading_strategy}_score_dict")(scores_dict)
+
     @classmethod
     def get_median_score_dict(cls, scores_dict):
         """Determine the median score in a dictionary of lists of scores
@@ -518,6 +545,36 @@ def get_median_score_dict(cls, scores_dict):
             median_scores[criterion] = criterion_score
         return median_scores
 
+    @classmethod
+    def get_mean_score_dict(cls, scores_dict):
+        """Determine the mean score in a dictionary of lists of scores
+
+        For a dictionary of lists, where each list contains a set of scores,
+        determine the mean value in each list.
+
+        Args:
+            scores_dict (dict): A dictionary of lists of int values. These int
+                values are reduced to a single value that represents the mean.
+
+        Returns:
+            (dict): A dictionary with criterion name keys and mean score
+                values.
+
+        Examples:
+            >>> scores = {
+            >>>     "foo": [5, 6, 12, 16, 22, 53],
+            >>>     "bar": [5, 6, 12, 16, 22, 53, 102]
+            >>> }
+            >>> Assessment.get_mean_score_dict(scores)
+            {"foo": 19, "bar": 31}
+
+        """
+        mean_scores = {}
+        for criterion, criterion_scores in scores_dict.items():
+            criterion_score = Assessment.get_mean_score(criterion_scores)
+            mean_scores[criterion] = criterion_score
+        return mean_scores
+
     @staticmethod
     def get_median_score(scores):
         """Determine the median score in a list of scores
@@ -552,6 +609,28 @@ def get_median_score(scores):
             )
         return median_score
 
+    @staticmethod
+    def get_mean_score(scores):
+        """Calculate the mean score from a list of scores
+
+        Args:
+            scores (list): A list of int values. These int values
+                are reduced to a single value that represents the mean.
+
+        Returns:
+            (int): The mean score.
+
+        Examples:
+            >>> scores = [5, 6, 12, 16, 22, 53]
+            >>> Assessment.get_mean_score(scores)
+            19
+
+        """
+        total_criterion_scores = len(scores)
+        if total_criterion_scores == 0:
+            return 0
+        return math.ceil(sum(scores) / float(total_criterion_scores))
+
     @classmethod
     def scores_by_criterion(cls, assessments):
         """Create a dictionary of lists for scores associated with criterion