Merge pull request #6 from act-org/EBI-item-selection

Ebi item selection

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: RSCAT
 Title: Shadow-Test Approach to Computerized Adaptive Testing
-Version: 1.0.2
+Version: 1.1.0
 Authors@R: c(
   person("Bingnan", "Jiang", email = "bnjiangece@gmail.com", role = c("aut", "cre")),
   person("ACT, Inc.", role = "cph")
@@ -21,7 +21,7 @@ Description: As an advanced approach to computerized adaptive testing (CAT),
 Depends: R (>= 3.4.0), rJava, shiny, shinycssloaders, shinyjs
 License: CC BY-NC 4.0
 Encoding: UTF-8
-RoxygenNote: 6.1.1
+RoxygenNote: 7.0.2
 Imports: 
     Metrics, ggplot2, gridExtra, grid, methods, stats, utils
 Collate: 

NAMESPACE

@@ -22,6 +22,7 @@ importFrom(shinyjs,disabled)
 importFrom(shinyjs,enable)
 importFrom(shinyjs,reset)
 importFrom(shinyjs,useShinyjs)
+importFrom(stats,quantile)
 importFrom(stats,sd)
 importFrom(utils,download.file)
 importFrom(utils,read.csv)

R/SimResult.R

@@ -26,6 +26,9 @@
 #' @slot shadowTests a list of length \code{numExaminees}. Each element of
 #'   the list is also a list representing the shadow test assembled at each 
 #'   adaptive stage.
+#' @slot engineTime a list of length \code{numExaminees}. Each element of
+#'   the list is a numeric vector representing the engine time at each adaptive step.
+#'	 the engine time includes time consumed by CAT algorithms and shadow test assembly.
 SimResult <- setClass(
   "SimResult",
   slots = c(
@@ -37,5 +40,6 @@ SimResult <- setClass(
             estThetaSEs = "list",
             scores = "list",
             itemsAdministered = "list",
-            shadowTests = "list")
+            shadowTests = "list",
+			      engineTime = "list")
 )

R/configClasses.R

@@ -38,6 +38,7 @@ SolverConfig <- setClass(
 #' @slot initialTheta the initial ability theta value.
 #' @slot scalingConstant the constraint to scale a discrimination coefficient.
 #'   estimated with the logistic model to the normal metric.
+#' @slot itemSelectionMethod  a character string specifying the item selection method.
 #' @slot scoreMethodConfig a rJava \code{jobjRef} object for CAT scoring method
 #'   configuration. It is generated by the function \code{scoreMethodConfig}.
 #' @slot exposureControlType a character string specifying the exposure control
@@ -54,6 +55,7 @@ CATConfig <- setClass(
     solverConfig = "SolverConfig", 
     initialTheta = "numeric", 
     scalingConstant = "numeric",
+    itemSelectionMethod = "character",
     scoreMethodConfig = "jobjRef",
     exposureControlType = "character",
     exposureControlRate = "numeric",
@@ -63,6 +65,7 @@ CATConfig <- setClass(
     solverConfig = SolverConfig(), 
     initialTheta = 0.0, 
     scalingConstant = 1.0,
+    itemSelectionMethod = "maxInfo",
     scoreMethodConfig = rJava::.jnull(),
     exposureControlType = "None",
     exposureControlRate = 0.5,

R/runSim.R

@@ -63,10 +63,11 @@ runSim <- function(catConfig, testConfig, simConfig) {
       method = "runSim",
       rJava::.jcast(
         catConfig@scoreMethodConfig,
-        "org/act/cat/ScoringMethodConfig"
+        "org/act/cat/AbstractScoringMethodConfig"
       ),
       catConfig@initialTheta,
       catConfig@scalingConstant,
+      catConfig@itemSelectionMethod,
       catConfig@exposureControlType,
       catConfig@exposureControlRate,
       catConfig@lValue,
@@ -101,6 +102,7 @@ runSim <- function(catConfig, testConfig, simConfig) {
   scoresList <- list()
   itemAdministeredList <- list()
   shadowTestsList <- list()
+  engineTimeList <- list()
 
   # Retrive data from the Java object
   for (i in 1:numExaminees) {
@@ -113,6 +115,7 @@ runSim <- function(catConfig, testConfig, simConfig) {
     scores <- numeric(testConfig@testLength)
     itemsAdmin <- character(testConfig@testLength)
     shadowTestExamineeList <- list()
+	  engineTime <- numeric(testConfig@testLength)
 
     for (j in 1:testConfig@testLength) {
 
@@ -131,12 +134,15 @@ runSim <- function(catConfig, testConfig, simConfig) {
       resultExamineeStage <- resultExaminee$getShadowTestList()$get(as.integer(j - 1))
       shadowTestExamineeList[[j]] <- 
         unlist(strsplit(gsub("\\[|\\]", "", resultExamineeStage$toString()), ", "))
+	    engineTime[j] <-
+        resultExaminee$getCatEngineTimeList()$get(as.integer(j-1))
     }
     estThetasList[[i]] <- thetas
     estThetaSEsList[[i]] <- thetaSEs
     scoresList[[i]] <- scores
     itemAdministeredList[[i]] <- itemsAdmin
     shadowTestsList[[i]] <- shadowTestExamineeList
+    engineTimeList[[i]] <- engineTime
   }
   return(
     SimResult(
@@ -148,7 +154,8 @@ runSim <- function(catConfig, testConfig, simConfig) {
       estThetaSEs = estThetaSEsList,
       scores = scoresList,
       itemsAdministered = itemAdministeredList,
-      shadowTests = shadowTestsList
+      shadowTests = shadowTestsList,
+      engineTime = engineTimeList
     )
   )
 

R/shinyAppServer.R

@@ -94,6 +94,7 @@ shinyAppServer <- function(input, output) {
       # CAT configuration
       catConfig <- CATConfig(
         solverConfig = solverConfig,
+        itemSelectionMethod = input$itemSelectionMethod,
         scoreMethodConfig = scoreMethodConfig(eapConfig),
         lValue = input$lValue,
         exposureControlType = input$exposureType,
@@ -288,7 +289,7 @@ shinyAppServer <- function(input, output) {
     output$simResultSummary <- NULL
     resetElementIds <- c("testName", "testLength", "itemPoolFile", 
       "passagePoolFile", "minNumPassage", "maxNumPassage", "constraintFile", 
-      "enemyItmesStr", "scoringMethod", "scoringAdvanced",
+      "enemyItmesStr", "itemSelectionMethod", "scoringMethod", "scoringAdvanced",
       "numPoints", "minPoint", "maxPoint", "priorDist", "distMean", "distSd", 
       "aVal", "bVal", "exposureType", "goalRate", "simName", "simTestTakerNum",
       "trueThetePriorDistType", "thetaMin", "thetaMax", "thetaMean", "thetaSD",

R/shinyAppUI.R

@@ -75,6 +75,9 @@ shinyAppUI <- fluidPage(
       hr(),
       h4("CAT Algorithm Configuration"),
 
+      # Select item selection method
+      selectInput("itemSelectionMethod", "Item Selection Criterion", c("Max Information" = "maxInfo", "EBI" = "ebi")),
+
       # Select the scoring method
       selectInput("scoringMethod", "Scoring Method", c("EAP" = "eap")),
 

R/utilFunctions.R

@@ -2,20 +2,41 @@
 #' 
 #' @param object an object of \code{SimResult}.
 #' Generates the summary report of CAT simulation.
+#' @importFrom stats quantile
 #' @export
 setMethod(
   f = "summary",
   signature = c("SimResult"),
   definition = function(object) {
     thetaBias <- Metrics::bias(object@trueThetas, object@finalThetas)
     thetaRMSE <- Metrics::rmse(object@trueThetas, object@finalThetas)
+    engineTimeMean <- mean(unlist(object@engineTime))
+    engineTimeMin <- min(unlist(object@engineTime))
+    engineTimeMax <- max(unlist(object@engineTime))
+    engineTimeSd <- format(round(sd(unlist(object@engineTime)), 4), nsmall = 4)
     thetaBias <- format(round(thetaBias, 4), nsmall = 4)
     thetaRMSE <- format(round(thetaRMSE, 4), nsmall = 4)
-    metrics <- list(thetaBias = thetaBias, thetaRMSE = thetaRMSE)
+    quantileVal <- format(round(unname(quantile(unlist(object@engineTime),
+                          c(0.25, 0.5, 0.75, 0.99))), 3), nsmall = 3)
+
+    metrics <- list(thetaBias = thetaBias, thetaRMSE = thetaRMSE, 
+                    engineTimeMean = engineTimeMean,
+                    engineTimeMin = engineTimeMin,
+                    engineTimeMax = engineTimeMax,
+                    engineTimeSd = engineTimeSd)
     msg <- paste("Numer of simulated examinees: ", 
       object@numExaminees, "\n",
       "Theta estimate bias: ", thetaBias, "\n",
-      "Theta estimate RMSE: ", thetaRMSE, "\n", sep = "")
+      "Theta estimate RMSE: ", thetaRMSE, "\n",
+      "Mean engine time (step): ", engineTimeMean, "\n",
+      "Sd of engine time (step): ", engineTimeSd, "\n",
+      "Min engine time (step): ", engineTimeMin, "\n",
+      "Max engine time (step): ", engineTimeMax, "\n",
+      "Engine time 25% percentile (step): ", quantileVal[1], "\n",
+      "Engine time 50% percentile (step): ", quantileVal[2], "\n",
+      "Engine time 75% percentile (step): ", quantileVal[3], "\n",
+      "Engine time 99% percentile (step): ", quantileVal[4], "\n",
+      sep = "")
     cat(msg)
     ret <- list(msg = msg, metrics = metrics)
     return(ret)
@@ -67,6 +88,7 @@ result2CSV <- function(simResult, file) {
         "Score" = simResult@scores[[n]], 
         "Theta Estimate" = simResult@estThetas[[n]],
         "Theta Estimate SE" = simResult@estThetaSEs[[n]], 
+        "Engine Time" = simResult@engineTime[[n]],
         "Shadow Test" = sapply(simResult@shadowTests[[n]], pasteWithQuote))
       write.csv(examineeOverall[[n]], row.names = FALSE)
       write.csv(examineeStage[[n]], row.names = FALSE)

README.md

@@ -2,7 +2,7 @@
 
 ## Overview
 As an advanced approach to computerized adaptive testing (CAT), 
-shadow testing (van der Linden(2005) <doi:10.1007/0-387-29054-0>) dynamically assembles entire shadow tests as a part of 
+shadow testing (van der Linden, 2005, <doi:10.1007/0-387-29054-0>) dynamically assembles entire shadow tests as a part of 
 selecting items throughout the testing process. 
 Selecting items from shadow tests guarantees the compliance of all content 
 constraints defined by the blueprint. RSCAT is an R package for the 

java/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.act</groupId>
 	<artifactId>RSCAT</artifactId>
-	<version>1.0.2</version>
+	<version>1.1.0</version>
 	<developers>
 		<developer>
 			<id>jiangb</id>
@@ -59,7 +59,7 @@
 		<dependency>
 			<groupId>com.dashoptimization</groupId>
 			<artifactId>xprm</artifactId>
-			<version>5.0.0</version>
+			<version>5.0.2</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>

java/src/main/java/org/act/cat/ScoringMethodConfig.java → java/src/main/java/org/act/cat/AbstractScoringMethodConfig.java

@@ -1,48 +1,48 @@
-package org.act.cat;
-
-import org.act.util.ProbDistribution;
-
-/**
- * This class defines a scoring method configuration. Parameters in the
- * configuration are applied to the scoring algorithm.
- */
-public abstract class ScoringMethodConfig {
-
-    private final ProbDistribution priorDistribution;
-
-    /**
-     * Constructs an empty {@link ScoringMethodConfig}.
-     */
-    public ScoringMethodConfig() {
-        this.priorDistribution = null;
-    }
-
-    /**
-     * Constructs a new {@link ScoringMethodConfig}.
-     *
-     * @param priorDistribution the prior distribution used in the scoring
-     *            method.
-     * @see ProbDistribution
-     */
-    public ScoringMethodConfig(ProbDistribution priorDistribution) {
-        this.priorDistribution = priorDistribution;
-    }
-
-    /**
-     * Returns the type of the scoring method related to the configuration.
-     *
-     * @return the type of the scoring method
-     */
-    public abstract ScoringMethod.SUPPORTED_METHODS scoringMethod();
-
-    /**
-     * Returns the prior distribution used in the scoring method.
-     *
-     * @return the prior distribution
-     * @see ProbDistribution
-     */
-    public ProbDistribution getPriorDistribution() {
-        return priorDistribution;
-    }
-
-}
+package org.act.cat;
+
+import org.act.util.ProbDistribution;
+
+/**
+ * This class defines a scoring method configuration. Parameters in the
+ * configuration are applied to the scoring algorithm.
+ */
+public abstract class AbstractScoringMethodConfig {
+
+    private final ProbDistribution priorDistribution;
+
+    /**
+     * Constructs an empty {@link AbstractScoringMethodConfig}.
+     */
+    public AbstractScoringMethodConfig() {
+        this.priorDistribution = null;
+    }
+
+    /**
+     * Constructs a new {@link AbstractScoringMethodConfig}.
+     *
+     * @param priorDistribution the prior distribution used in the scoring
+     *            method.
+     * @see ProbDistribution
+     */
+    public AbstractScoringMethodConfig(ProbDistribution priorDistribution) {
+        this.priorDistribution = priorDistribution;
+    }
+
+    /**
+     * Returns the type of the scoring method related to the configuration.
+     *
+     * @return the type of the scoring method
+     */
+    public abstract ScoringMethod.SUPPORTED_METHODS scoringMethod();
+
+    /**
+     * Returns the prior distribution used in the scoring method.
+     *
+     * @return the prior distribution
+     * @see ProbDistribution
+     */
+    public ProbDistribution getPriorDistribution() {
+        return priorDistribution;
+    }
+
+}

java/src/main/java/org/act/cat/CatConfig.java

@@ -35,7 +35,7 @@ public interface CatConfig {
      *
      * @return the instance of {@link ScoringMethodConfig}
      */
-    ScoringMethodConfig scoringMethodConfig();
+    AbstractScoringMethodConfig scoringMethodConfig();
 
     /**
      * Returns the number <code>L</code> of items that are randomly administered at
@@ -54,4 +54,12 @@ public interface CatConfig {
      * @return the exposure control configuration
      */
     ExposureControlConfig exposureControlConfig();
+
+    /**
+     * Returns the type of item selection method.
+     *
+     * @return the type of item selection method.
+     * @see ItemSelectionMethod.SUPPORTED_METHODS
+     */
+    ItemSelectionMethod.SUPPORTED_METHODS itemSelectionMethod();
 }