One booklet 2PL and GPC items

We examined item parameter recovery under the following conditions: 2 (IRT models) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 1 (test booklet)

Two types of IRT models were used: 2PL items and generalized partial credit (GPC) items
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to b_bounds = (-2, 2) where -2 is the lowest generating value and 2 is the highest generating value
- The bounds of the item discrimination parameter, a, are constrained to a_bounds = (0.75, 1.25) where 0.75 is the lowest generating value and 1.25 is the highest generating value
Two IRT R packages were evaluated: TAM (version 2.4-9) and mirt (version 1.25)
Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
Four test lengths were used: 40, 60, 80, and 100
A single booklet was used

One hundred replications were used for each condition for the calibration

Summary of item parameter recovery:
- TAM and mirt demonstrated a similar level of accuracy
- For 2PL items:
  - b-parameter recovered well, with correlation ranging from 0.989 to 0.999, with bias ranging from -0.009 to 0.032, and with RMSE ranging from 0.051 to 0.245
  - a-parameter recovered moderately, with correlation ranging from 0.763 to 0.971, with bias ranging from -0.063 to 0.015, and with RMSE ranging from 0.044 to 0.156
- For GPC items:
  - b1-parameter recovered well, with correlation ranging from 0.974 to 0.999, with bias ranging from -0.053 to -0.002, and with RMSE ranging from 0.047 to 0.178
  - b2-parameter recovered well, with correlation ranging from 0.984 to 0.999, with bias ranging from -0.009 to 0.019, and with RMSE ranging from 0.046 to 0.176
  - a-parameter recovered moderately, with correlation ranging from 0.819 to 0.981, with bias ranging from -0.060 to 0.011, and with RMSE ranging from 0.032 to 0.110
- In general:
  - Sample sizes of 5000 consistently produced the most accurate results
  - Four levels of test lengths performed very similarly

# Load libraries
if(!require(lsasim)){  
  install.packages("lsasim")
  library(lsasim) #version 1.0.1
}

if(!require(mirt)){  
  install.packages("mirt")
  library(mirt) #version 1.25
}

if(!require(TAM)){
  install.packages("TAM")
  library(TAM) #version 2.4-9
}

# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items 
K.cond  <- 1                 #number of booklets  

# Set up number of replications
reps <- 100

# Create space for outputs
results <- NULL

#==============================================================================#
# START SIMULATION
#==============================================================================#

for (N in N.cond) { #sample size
  
  for (I in I.cond) { #number of total items
    
    # generate item parameters for 2PL and GPC models
    set.seed(4374) # fix item parameters across replications
    item_pool <- gen2PL_GPC <- lsasim::item_gen(n_2pl = c(I/2, I/2), 
                                                thresholds = c(1, 2), 
                                                b_bounds = c(-2, 2), 
                                                a_bounds = c(0.75, 1.25)) 
    
    for (K in K.cond) { #number of booklets
      
      for (r in 1:reps) { #number of replications
        
        #------------------------------------------------------------------------------#
        # Data simulation
        #------------------------------------------------------------------------------#
        
        set.seed(8088*(r+1))
        
        # generate thetas
        theta <- rnorm(N, mean=0, sd=1)
        
        # assign items to block
        block_bk1 <- lsasim::block_design(n_blocks = K, 
                                          item_parameters = item_pool)
        
        #assign block to booklet
        book_bk1 <- lsasim::booklet_design(item_block_assignment = 
                                             block_bk1$block_assignment,
                                           book_design = matrix(K))
        #assign booklet to subjects
        book_samp <- lsasim::booklet_sample(n_subj = N, 
                                            book_item_design = book_bk1, 
                                            book_prob = NULL)
        
        # generate item responses 
        cog <- lsasim::response_gen(subject = book_samp$subject, 
                                    item = book_samp$item, 
                                    theta = theta, 
                                    a_par = item_pool$a,
                                    b_par = item_pool$b,
                                    d_par = list(item_pool$d1,
                                                 item_pool$d2))
        
        # extract item responses (excluding "subject" column)
        resp <- cog[, c(1:I)]
        
        #------------------------------------------------------------------------------#
        # Item calibration
        #------------------------------------------------------------------------------#
        
        # fit 2PL and GPC models using mirt package
        mirt.mod <- NULL
        mirt.mod <- mirt::mirt(resp, 1, 
                               itemtype = c(rep('2PL', I/2), rep('gpcm', I/2)),
                               verbose = F)
        
        # fit 2PL and GPC models using TAM package
        tam.mod <- NULL
        tam.mod <- TAM::tam.mml.2pl(resp, irtmodel = "GPCM", 
                                    control = list(maxiter = 200))
        
        #------------------------------------------------------------------------------#
        # Item parameter extraction
        #------------------------------------------------------------------------------#
        
        # extract item parameters in mirt package
        mirt.mod.par <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items
        mirt.mod.par_2PL <- mirt.mod.par[c(1:I/2),]
        mirt.mod.par_GPC <-mirt.mod.par[c((1/2*I+1):I),]
        
        #--- 2PL items
        mirt_2PL_b <- mirt.mod.par_2PL[,"b"]
        mirt_2PL_a <- mirt.mod.par_2PL[,"a"]
        
        #--- GPC items
        mirt_GPC_b1 <- mirt.mod.par_GPC[,"b1"]
        mirt_GPC_b2 <- mirt.mod.par_GPC[,"b2"]
        mirt_GPC_a <- mirt.mod.par_GPC[,"a"]

        # extract item parameters in TAM package
        tam.mod.par_2PL <- tam.mod$item[c(1:I/2),]
        tam.mod.par_GPC <- tam.mod$item[c((1/2*I+1):I),]
        
        # convert TAM output into GPCM parametrization
        #--- 2PL items
        tam_2PL_b <- tam.mod.par_2PL$AXsi_.Cat1 / tam.mod.par_2PL$B.Cat1.Dim1
        tam_2PL_a <- tam.mod.par_2PL$B.Cat1.Dim1
        
        #--- GPC items
        tam_GPC_b1 <- (tam.mod.par_GPC$AXsi_.Cat1 / tam.mod.par_GPC$B.Cat1.Dim1)
        tam_GPC_b2 <- (tam.mod.par_GPC$AXsi_.Cat2 / tam.mod.par_GPC$B.Cat1.Dim1) 
                       -  tam_GPC_b1
        tam_GPC_a <- tam.mod.par_GPC$B.Cat1.Dim1

        #------------------------------------------------------------------------------#
        # Item parameter recovery
        #------------------------------------------------------------------------------#
        
        # summarize results
        itempars <- data.frame(matrix(c(N, I, K, r), nrow = 1))
        colnames(itempars) <- c("N", "I", "K", "rep")

        # retrieve generated item parameters    
        #--- 2PL items
        gen2PL_b <- item_pool[c(1:I/2), "b"]
        gen2PL_a <- item_pool[c(1:I/2), "a"]

        #--- GPC items
        genGPC_b1 <- item_pool[c((1/2*I+1):I), "b"] + item_pool[c((1/2*I+1):I), "d1"]
        genGPC_b2 <- item_pool[c((1/2*I+1):I), "b"] + item_pool[c((1/2*I+1):I), "d2"]
        genGPC_a <- item_pool[c((1/2*I+1):I), "a"]
        
        # calculate corr, bias, RMSE for item parameters from mirt 
        #--- 2PL items
        itempars$corr_mirt_2PL_b <- cor( gen2PL_b, mirt_2PL_b)
        itempars$bias_mirt_2PL_b <- mean( mirt_2PL_b - gen2PL_b)
        itempars$RMSE_mirt_2PL_b <- sqrt(mean( ( mirt_2PL_b - gen2PL_b)^2 )) 
        
        itempars$corr_mirt_2PL_a <- cor( gen2PL_a, mirt_2PL_a)
        itempars$bias_mirt_2PL_a <- mean( mirt_2PL_a - gen2PL_a)
        itempars$RMSE_mirt_2PL_a <- sqrt(mean( ( mirt_2PL_a - gen2PL_a)^2 )) 

        #--- GPC items
        itempars$corr_mirt_GPC_b1 <- cor( genGPC_b1, mirt_GPC_b1)
        itempars$bias_mirt_GPC_b1 <- mean( mirt_GPC_b1 - genGPC_b1)
        itempars$RMSE_mirt_GPC_b1 <- sqrt(mean( ( mirt_GPC_b1 - genGPC_b1)^2 )) 
        
        itempars$corr_mirt_GPC_b2 <- cor( genGPC_b2, mirt_GPC_b2)
        itempars$bias_mirt_GPC_b2 <- mean( mirt_GPC_b2 - genGPC_b2)
        itempars$RMSE_mirt_GPC_b2 <- sqrt(mean( ( mirt_GPC_b2 - genGPC_b2)^2 )) 
        
        itempars$corr_mirt_GPC_a <- cor( genGPC_a, mirt_GPC_a)
        itempars$bias_mirt_GPC_a <- mean( mirt_GPC_a - genGPC_a)
        itempars$RMSE_mirt_GPC_a <- sqrt(mean( ( mirt_GPC_a - genGPC_a)^2 )) 
        
        # calculate corr, bias, RMSE for item parameters from TAM
        #--- 2PL items
        itempars$corr_tam_2PL_b <- cor( gen2PL_b, tam_2PL_b)
        itempars$bias_tam_2PL_b <- mean( tam_2PL_b - gen2PL_b)
        itempars$RMSE_tam_2PL_b <- sqrt(mean( ( tam_2PL_b - gen2PL_b)^2 )) 
        
        itempars$corr_tam_2PL_a <- cor( gen2PL_a, tam_2PL_a)
        itempars$bias_tam_2PL_a <- mean( tam_2PL_a - gen2PL_a)
        itempars$RMSE_tam_2PL_a <- sqrt(mean( ( tam_2PL_a - gen2PL_a)^2 )) 
        
        #--- GPC items
        itempars$corr_tam_GPC_b1 <- cor( genGPC_b1, tam_GPC_b1)
        itempars$bias_tam_GPC_b1 <- mean( tam_GPC_b1 - genGPC_b1)
        itempars$RMSE_tam_GPC_b1 <- sqrt(mean( ( tam_GPC_b1 - genGPC_b1)^2 )) 
        
        itempars$corr_tam_GPC_b2 <- cor( genGPC_b2, tam_GPC_b2)
        itempars$bias_tam_GPC_b2 <- mean( tam_GPC_b2 - genGPC_b2)
        itempars$RMSE_tam_GPC_b2 <- sqrt(mean( ( tam_GPC_b2 - genGPC_b2)^2 )) 
        
        itempars$corr_tam_GPC_a <- cor( genGPC_a, tam_GPC_a)
        itempars$bias_tam_GPC_a <- mean( tam_GPC_a - genGPC_a)
        itempars$RMSE_tam_GPC_a <- sqrt(mean( ( tam_GPC_a - genGPC_a)^2 )) 
         
        #combine results
        results <- rbind(results, itempars)
        
      }
    }
  }
}

Correlation, bias, and RMSE for item parameter recovery in mirt package

#--- 2PL items
mirt_recovery_2PL <- aggregate(cbind(corr_mirt_2PL_b, bias_mirt_2PL_b, RMSE_mirt_2PL_b,
                                     corr_mirt_2PL_a, bias_mirt_2PL_a, RMSE_mirt_2PL_a) 
                               ~ N + I, 
                            data=results, mean, na.rm=TRUE)
names(mirt_recovery_2PL) <- c("Sample Size", "Test Length", 
                         "corr_b", "bias_b", "RMSE_b", 
                         "corr_a", "bias_a", "RMSE_a")
round(mirt_recovery_2PL, 3)

##    Sample Size Test Length corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1          500          40  0.990  0.004  0.187  0.763  0.015  0.156
## 2         1000          40  0.995 -0.005  0.125  0.866  0.002  0.105
## 3         5000          40  0.999 -0.004  0.055  0.969  0.000  0.047
## 4          500          60  0.990 -0.009  0.162  0.798  0.010  0.138
## 5         1000          60  0.995 -0.005  0.112  0.879  0.006  0.098
## 6         5000          60  0.999 -0.005  0.051  0.971  0.000  0.044
## 7          500          80  0.990  0.000  0.207  0.786  0.008  0.147
## 8         1000          80  0.995 -0.007  0.137  0.873  0.004  0.101
## 9         5000          80  0.999 -0.002  0.061  0.971 -0.001  0.045
## 10         500         100  0.989  0.001  0.209  0.766  0.013  0.144
## 11        1000         100  0.995 -0.008  0.140  0.858  0.008  0.101
## 12        5000         100  0.999 -0.004  0.063  0.966  0.000  0.045

#--- GPC items
mirt_recovery_GPC <- aggregate(cbind(corr_mirt_GPC_b1, bias_mirt_GPC_b1, RMSE_mirt_GPC_b1,
                                    corr_mirt_GPC_b2, bias_mirt_GPC_b2, RMSE_mirt_GPC_b2,
                                    corr_mirt_GPC_a, bias_mirt_GPC_a, RMSE_mirt_GPC_a) 
                               ~ N + I, 
                            data=results, mean, na.rm=TRUE)
names(mirt_recovery_GPC) <- c("Sample Size", "Test Length", 
                         "corr_b1", "bias_b1", "RMSE_b1", 
                         "corr_b2", "bias_b2", "RMSE_b2",
                         "corr_a", "bias_a", "RMSE_a")
round(mirt_recovery_GPC, 3)

##    Sample Size Test Length corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1          500          40   0.983  -0.011   0.158   0.988  -0.002   0.163  0.846  0.010  0.110
## 2         1000          40   0.992  -0.004   0.108   0.994  -0.009   0.114  0.907  0.006  0.079
## 3         5000          40   0.998  -0.003   0.049   0.999  -0.003   0.050  0.980  0.001  0.034
## 4          500          60   0.988  -0.003   0.159   0.988  -0.006   0.161  0.827  0.009  0.110
## 5         1000          60   0.993  -0.007   0.113   0.994  -0.004   0.113  0.905  0.005  0.076
## 6         5000          60   0.999  -0.006   0.048   0.999  -0.003   0.049  0.979  0.000  0.034
## 7          500          80   0.980  -0.007   0.148   0.987  -0.004   0.158  0.820  0.011  0.107
## 8         1000          80   0.990  -0.010   0.102   0.994  -0.008   0.108  0.895  0.006  0.074
## 9         5000          80   0.998  -0.005   0.047   0.999  -0.004   0.048  0.976  0.000  0.033
## 10         500         100   0.974  -0.008   0.151   0.984  -0.008   0.149  0.836  0.011  0.107
## 11        1000         100   0.987  -0.011   0.106   0.992  -0.009   0.105  0.910  0.005  0.074
## 12        5000         100   0.997  -0.005   0.047   0.998  -0.005   0.046  0.981  0.001  0.032

Correlation, bias, and RMSE for item parameter recovery in TAM package

#--- 2PL items
tam_recovery_2PL <- aggregate(cbind(corr_tam_2PL_b, bias_tam_2PL_b, RMSE_tam_2PL_b,
                                     corr_tam_2PL_a, bias_tam_2PL_a, RMSE_tam_2PL_a) 
                              ~ N + I, 
                               data=results, mean, na.rm=TRUE)
names(tam_recovery_2PL) <- c("Sample Size", "Test Length", 
                         "corr_b", "bias_b", "RMSE_b", 
                         "corr_a", "bias_a", "RMSE_a")
round(tam_recovery_2PL, 3)

##    Sample Size Test Length corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1          500          40  0.990  0.007  0.187  0.763  0.013  0.155
## 2         1000          40  0.995 -0.003  0.126  0.866  0.000  0.105
## 3         5000          40  0.999 -0.001  0.055  0.969 -0.003  0.047
## 4          500          60  0.990 -0.005  0.171  0.798 -0.005  0.135
## 5         1000          60  0.995 -0.002  0.115  0.879 -0.006  0.096
## 6         5000          60  0.999 -0.002  0.055  0.971 -0.016  0.046
## 7          500          80  0.990  0.015  0.228  0.785 -0.023  0.144
## 8         1000          80  0.995  0.002  0.154  0.873 -0.028  0.101
## 9         5000          80  0.999  0.003  0.081  0.971 -0.035  0.055
## 10         500         100  0.989  0.032  0.245  0.764 -0.039  0.142
## 11        1000         100  0.995  0.008  0.178  0.857 -0.051  0.108
## 12        5000         100  0.999  0.019  0.111  0.965 -0.063  0.076

#--- GPC items
tam_recovery_GPC <- aggregate(cbind(corr_tam_GPC_b1, bias_tam_GPC_b1, RMSE_tam_GPC_b1,
                                     corr_tam_GPC_b2, bias_tam_GPC_b2, RMSE_tam_GPC_b2,
                                     corr_tam_GPC_a, bias_tam_GPC_a, RMSE_tam_GPC_a) 
                              ~ N + I, 
                               data=results, mean, na.rm=TRUE)
names(tam_recovery_GPC) <- c("Sample Size", "Test Length", 
                         "corr_b1", "bias_b1", "RMSE_b1", 
                         "corr_b2", "bias_b2", "RMSE_b2",
                         "corr_a", "bias_a", "RMSE_a")
round(tam_recovery_GPC, 3)

##    Sample Size Test Length corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1          500          40   0.983  -0.009   0.158   0.988   0.001   0.165  0.845  0.008  0.109
## 2         1000          40   0.992  -0.003   0.108   0.994  -0.007   0.115  0.907  0.003  0.078
## 3         5000          40   0.998  -0.002   0.049   0.999  -0.001   0.050  0.980 -0.001  0.034
## 4          500          60   0.988  -0.005   0.169   0.988   0.003   0.168  0.827 -0.005  0.107
## 5         1000          60   0.993  -0.009   0.116   0.994   0.004   0.115  0.904 -0.007  0.075
## 6         5000          60   0.999  -0.009   0.052   0.999   0.006   0.053  0.979 -0.015  0.036
## 7          500          80   0.980  -0.012   0.166   0.987   0.019   0.175  0.819 -0.019  0.105
## 8         1000          80   0.990  -0.022   0.115   0.994   0.009   0.121  0.895 -0.027  0.076
## 9         5000          80   0.998  -0.023   0.059   0.999   0.009   0.061  0.976 -0.033  0.046
## 10         500         100   0.974  -0.029   0.178   0.984   0.017   0.176  0.834 -0.040  0.108
## 11        1000         100   0.987  -0.053   0.141   0.992   0.002   0.135  0.909 -0.052  0.087
## 12        5000         100   0.997  -0.045   0.080   0.998   0.012   0.074  0.980 -0.060  0.067

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

One booklet 2PL and GPC items

Clone this wiki locally