-
Notifications
You must be signed in to change notification settings - Fork 5
One booklet 3PL items
tmatta edited this page Oct 17, 2017
·
3 revisions
We examined item parameter recovery under the following conditions: 1 (IRT model) x 3 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 1 (test booklet)
- One IRT model was included: 3PL model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value - The bounds of the item guessing parameter, c, are constrained to
c_bounds = (0, 0.25)where 0 is the lowest generating value and 0.25 is the highest generating value
- Three IRT R packages were evaluated:
TAM(version 2.4-9),mirt(version 1.25), andltm(version, 1.0-0) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- A single booklet was used.
- One hundred replications were used for each condition for the calibration
- Summary of item parameter recovery:
-
TAMdemonstrated slightly better results, butmirtandltmseemed to perform similarly - For a- and c-parameters, the corrlations between true and estimated parameters were low, but RMSE and bias were within the acceptive level
- For b-parameter, the correlation values between the true and estimated parameters ranged from 0.887 to 0.988, the bias values were in the range of 0.061 to 0.287, and the RMSE values ranged from 0.205 to 0.661
- For a-parameter, the correlation values between the true and estimated parameters ranged from 0.116 to 0.496, the bias values were in the range of -0.209 to 0.203, and the RMSE values ranged from 0.241 to 0.740
- For c-parameter, the correlation values between the true and estimated parameters ranged from 0.114 to 0.418, the bias values were in the range of -0.075 to -0.013, and the RMSE values ranged from 0.089 to 0.137
- For all item parameters; when sample size increased, recovery accuracy improved further
-
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
if(!require(ltm)){
install.packages("ltm")
library(ltm) #version 1.0-0
}# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- 1 #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of itemS
# generate item parameters for a 3PL model
set.seed(4365) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_3pl = I, thresholds = 1,
b_bounds = c(-2, 2),
a_bounds = c(0.75, 1.25),
c_bounds = c(0, .25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
#------------------------------------------------------------------------------#
# Data simulation
#------------------------------------------------------------------------------#
set.seed(8088*(r+3))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to block
block_bk1 <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool)
#assign block to booklet
book_bk1 <- lsasim::booklet_design(item_block_assignment =
block_bk1$block_assignment,
book_design = matrix(K))
#assign booklet to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = book_bk1,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject, item = book_samp$item,
theta = theta,
b_par = item_pool$b,
a_par = item_pool$a,
c_par = item_pool$c)
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit 3PL model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = '3PL', verbose = F,
technical = list( NCYCLES = 500))
# fit 3PL model using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml.3pl(resp, est.guess = c(1:I),
control = list(maxiter = 200))
# fit 3PL model using ltm package
ltm.mod <- NULL
ltm.mod <- ltm::tpm(resp, IRT.param=T, control = list(iter.qN = 1000))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b, a, c in mirt package
mirt_b <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b"]
mirt_a <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"a"]
mirt_c <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"g"]
# convert TAM output into 3PL parametrization
tam_b <- (tam.mod$item$AXsi_.Cat1) / (tam.mod$item$B.Cat1.Dim1)
tam_a <- (tam.mod$item$B.Cat1.Dim1)
tam_c <- tam.mod$item$guess
# extract Dffclt, Dscrmn, Gussng in ltm package
ltm_b <- data.frame(coef(ltm.mod))$Dffclt
ltm_a <- data.frame(coef(ltm.mod))$Dscrmn
ltm_c <- data.frame(coef(ltm.mod))$Gussng
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow=1))
colnames(itempars) <- c("N", "I", "K", "rep")
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b <- cor( item_pool$b, mirt_b)
itempars$bias_mirt_b <- mean( mirt_b - item_pool$b )
itempars$RMSE_mirt_b <- sqrt(mean( ( mirt_b - item_pool$b)^2 ))
itempars$corr_mirt_a <- cor( item_pool$a, mirt_a)
itempars$bias_mirt_a <- mean( mirt_a - item_pool$a )
itempars$RMSE_mirt_a <- sqrt(mean( ( mirt_a - item_pool$a)^2 ))
itempars$corr_mirt_c <- cor( item_pool$c, mirt_c)
itempars$bias_mirt_c <- mean( mirt_c - item_pool$c )
itempars$RMSE_mirt_c <- sqrt(mean( ( mirt_c - item_pool$c)^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b <- cor( item_pool$b, tam_b)
itempars$bias_tam_b <- mean( tam_b - item_pool$b )
itempars$RMSE_tam_b <- sqrt(mean( ( tam_b - item_pool$b)^2 ))
itempars$corr_tam_a <- cor( item_pool$a, tam_a)
itempars$bias_tam_a <- mean( tam_a - item_pool$a )
itempars$RMSE_tam_a <- sqrt(mean( ( tam_a - item_pool$a)^2 ))
itempars$corr_tam_c <- cor( item_pool$c, tam_c)
itempars$bias_tam_c <- mean( tam_c - item_pool$c )
itempars$RMSE_tam_c <- sqrt(mean( ( tam_c - item_pool$c)^2 ))
# calculate corr, bias, RMSE for item parameters in ltm pacakge
itempars$corr_ltm_b <- cor( item_pool$b, ltm_b)
itempars$bias_ltm_b <- mean( ltm_b - item_pool$b )
itempars$RMSE_ltm_b <- sqrt(mean( ( ltm_b - item_pool$b)^2 ))
itempars$corr_ltm_a <- cor( item_pool$a, ltm_a)
itempars$bias_ltm_a <- mean( ltm_a - item_pool$a )
itempars$RMSE_ltm_a <- sqrt(mean( ( ltm_a - item_pool$a)^2 ))
itempars$corr_ltm_c <- cor( item_pool$c, ltm_c)
itempars$bias_ltm_c <- mean( ltm_c - item_pool$c )
itempars$RMSE_ltm_c <- sqrt(mean( ( ltm_c - item_pool$c)^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b, bias_mirt_b, RMSE_mirt_b,
corr_mirt_a, bias_mirt_a, RMSE_mirt_a,
corr_mirt_c, bias_mirt_c, RMSE_mirt_c) ~ N + I,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("Sample Size", "Test Length",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a",
"corr_c", "bias_c", "RMSE_c")
round(mirt_recovery, 3)## Sample Size Test Length corr_b bias_b RMSE_b corr_a bias_a RMSE_a corr_c bias_c RMSE_c
## 1 500 40 0.892 0.251 0.654 0.175 -0.043 0.527 0.209 -0.024 0.133
## 2 1000 40 0.936 0.191 0.509 0.301 -0.138 0.353 0.230 -0.041 0.117
## 3 5000 40 0.985 0.115 0.297 0.487 -0.206 0.286 0.407 -0.064 0.094
## 4 500 60 0.887 0.256 0.618 0.231 -0.046 0.413 0.158 -0.016 0.134
## 5 1000 60 0.933 0.206 0.485 0.306 -0.123 0.314 0.214 -0.034 0.116
## 6 5000 60 0.984 0.142 0.297 0.473 -0.186 0.270 0.314 -0.055 0.096
## 7 500 80 0.892 0.286 0.640 0.147 -0.029 0.492 0.169 -0.018 0.136
## 8 1000 80 0.937 0.235 0.496 0.233 -0.123 0.349 0.202 -0.037 0.121
## 9 5000 80 0.983 0.164 0.325 0.386 -0.191 0.286 0.379 -0.059 0.099
## 10 500 100 0.894 0.287 0.648 0.120 -0.022 0.491 0.150 -0.013 0.134
## 11 1000 100 0.939 0.229 0.515 0.211 -0.108 0.349 0.194 -0.030 0.116
## 12 5000 100 0.983 0.173 0.355 0.349 -0.178 0.287 0.329 -0.051 0.094
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b, bias_tam_b, RMSE_tam_b,
corr_tam_a, bias_tam_a, RMSE_tam_a,
corr_tam_c, bias_tam_c, RMSE_tam_c) ~ N + I,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("Sample Size", "Test Length",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a",
"corr_c", "bias_c", "RMSE_c")
round(tam_recovery, 3)## Sample Size Test Length corr_b bias_b RMSE_b corr_a bias_a RMSE_a corr_c bias_c RMSE_c
## 1 500 40 0.914 0.149 0.527 0.219 0.194 0.602 0.155 -0.040 0.127
## 2 1000 40 0.953 0.111 0.395 0.318 0.097 0.416 0.173 -0.055 0.116
## 3 5000 40 0.988 0.061 0.232 0.496 0.029 0.257 0.346 -0.075 0.103
## 4 500 60 0.914 0.151 0.493 0.266 0.203 0.551 0.124 -0.030 0.130
## 5 1000 60 0.954 0.128 0.367 0.322 0.081 0.376 0.176 -0.044 0.115
## 6 5000 60 0.987 0.087 0.205 0.483 -0.006 0.241 0.266 -0.065 0.104
## 7 500 80 0.899 0.198 0.580 0.177 0.132 0.565 0.134 -0.033 0.135
## 8 1000 80 0.952 0.153 0.393 0.247 0.045 0.403 0.168 -0.049 0.122
## 9 5000 80 0.986 0.107 0.225 0.401 -0.055 0.256 0.339 -0.070 0.107
## 10 500 100 0.914 0.191 0.542 0.149 0.074 0.527 0.114 -0.028 0.131
## 11 1000 100 0.953 0.156 0.413 0.231 -0.015 0.373 0.158 -0.043 0.117
## 12 5000 100 0.986 0.128 0.273 0.370 -0.122 0.270 0.294 -0.061 0.102
- Correlation, bias, and RMSE for item parameter recovery in
ltmpackage
ltm_recovery <- aggregate(cbind(corr_ltm_b, bias_ltm_b, RMSE_ltm_b,
corr_ltm_a, bias_ltm_a, RMSE_ltm_a,
corr_ltm_c, bias_ltm_c, RMSE_ltm_c) ~ N + I,
data=results, mean, na.rm=TRUE)
names(ltm_recovery) <- c("Sample Size", "Test Length",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a",
"corr_c", "bias_c", "RMSE_c")
round(ltm_recovery, 3)## Sample Size Test Length corr_b bias_b RMSE_b corr_a bias_a RMSE_a corr_c bias_c RMSE_c
## 1 500 40 0.889 0.240 0.661 0.168 -0.007 0.740 0.203 -0.027 0.136
## 2 1000 40 0.936 0.179 0.508 0.299 -0.138 0.367 0.224 -0.045 0.119
## 3 5000 40 0.985 0.103 0.302 0.488 -0.209 0.288 0.385 -0.068 0.099
## 4 500 60 0.887 0.241 0.622 0.224 -0.036 0.488 0.155 -0.020 0.135
## 5 1000 60 0.937 0.198 0.475 0.307 -0.125 0.314 0.211 -0.035 0.116
## 6 5000 60 0.986 0.147 0.292 0.472 -0.184 0.269 0.353 -0.054 0.092
## 7 500 80 0.891 0.277 0.655 0.140 -0.012 0.654 0.167 -0.022 0.137
## 8 1000 80 0.944 0.228 0.477 0.233 -0.127 0.350 0.209 -0.038 0.117
## 9 5000 80 0.984 0.172 0.320 0.384 -0.190 0.285 0.418 -0.057 0.094
## 10 500 100 0.899 0.269 0.647 0.116 -0.022 0.580 0.150 -0.017 0.133
## 11 1000 100 0.950 0.221 0.489 0.213 -0.119 0.349 0.206 -0.032 0.110
## 12 5000 100 0.985 0.184 0.348 0.351 -0.183 0.288 0.376 -0.049 0.089