-
Notifications
You must be signed in to change notification settings - Fork 5
One booklet GPC items
tmatta edited this page Oct 17, 2017
·
3 revisions
We examined item parameter recovery under the following conditions: 1 (IRT model) x 3 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 1 (test booklet)
- One IRT model was included: generalized partial credit (GPC) model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value
- Three IRT R packages were evaluated:
TAM(version 2.4-9),mirt(version 1.25), andltm(version, 1.0-0) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- A single booklet was used.
- One hundred replications were used for each condition for the calibration
- Summary of item parameter recovery:
-
TAM,mirt, andltmdemonstrated a similar level of accuracy - b1-parameter recovered well, with correlation ranging from 0.982 to 0.998, with bias ranging from -0.115 to -0.002, and with RMSE ranging from 0.045 to 0.266
- b2-parameter recovered well, with correlation ranging from 0.985 to 0.999, with bias ranging from -0.009 to 0.066, and with RMSE ranging from 0.047 to 0.257
- a-parameter recovered moderately, with correlation ranging from 0.787 to 0.976, with bias ranging from -0.158 to 0.011, and with RMSE ranging from 0.033 to 0.170
- For all parameters, sample sizes of 5000 consistently produced the most accurate results
- For b1- and b2-parameter, four levels of test lengths performed very similarly
- For a-parameter, four levels of test lengths performed similarly
-
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
if(!require(ltm)){
install.packages("ltm")
library(ltm) #version 1.0-0
}# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- 1 #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a GPCM model
set.seed(4367) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_2pl = I,
thresholds = 2,
b_bounds = c(-2, 2),
a_bounds = c(.75, 1.25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
#------------------------------------------------------------------------------#
# Data simulation
#------------------------------------------------------------------------------#
set.seed(8088*(r+5))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to block
block_bk1 <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool)
#assign block to booklet
book_bk1 <- lsasim::booklet_design(item_block_assignment
= block_bk1$block_assignment,
book_design = matrix(K))
#assign booklet to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = book_bk1,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
b_par = item_pool$b,
a_par = item_pool$a,
d_par = list(item_pool$d1,
item_pool$d2))
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit GPCM model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = "gpcm", verbose = F,
technical = list( NCYCLES = 500))
# fit GPCM model using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml.2pl(resp, irtmodel = "GPCM",
control = list(maxiter = 200))
# fit GPCM model using ltm package
ltm.mod <- NULL
ltm.mod <- ltm::gpcm(resp, IRT.param=T, control = list(iter.qN = 1000))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b1, b2, a in mirt package
mirt_b1 <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b1"]
mirt_b2 <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b2"]
mirt_a <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"a"]
# convert TAM output into GPCM parametrization
tam_b1 <- (tam.mod$item$AXsi_.Cat1/tam.mod$item$B.Cat1.Dim1)
tam_b2 <- (tam.mod$item$AXsi_.Cat2/tam.mod$item$B.Cat1.Dim1)
- (tam.mod$item$AXsi_.Cat1/tam.mod$item$B.Cat1.Dim1)
tam_a <- tam.mod$item$B.Cat1.Dim1
# extract Dffclt and Dscrmn in ltm package
ltm_b1 <- data.frame(coef(ltm.mod))$Catgr.1
ltm_b2 <- data.frame(coef(ltm.mod))$Catgr.2
ltm_a <- data.frame(coef(ltm.mod))$Dscrmn
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow=1))
colnames(itempars) <- c("N", "I", "K", "rep")
# retrieve generated item parameters
genGPC.b1 <- item_pool$b + item_pool$d1
genGPC.b2 <- item_pool$b + item_pool$d2
genGPC.a <- item_pool$a
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b1 <- cor( genGPC.b1, mirt_b1)
itempars$bias_mirt_b1 <- mean( mirt_b1 - genGPC.b1 )
itempars$RMSE_mirt_b1 <- sqrt(mean( ( mirt_b1 - genGPC.b1 )^2 ))
itempars$corr_mirt_b2 <- cor( genGPC.b2, mirt_b2)
itempars$bias_mirt_b2 <- mean( mirt_b2 - genGPC.b2 )
itempars$RMSE_mirt_b2 <- sqrt(mean( ( mirt_b2 - genGPC.b2 )^2 ))
itempars$corr_mirt_a <- cor( genGPC.a, mirt_a)
itempars$bias_mirt_a <- mean( mirt_a - genGPC.a )
itempars$RMSE_mirt_a <- sqrt( mean( ( mirt_a - genGPC.a )^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b1 <- cor( genGPC.b1, tam_b1)
itempars$bias_tam_b1 <- mean( tam_b1 - genGPC.b1 )
itempars$RMSE_tam_b1 <- sqrt(mean( ( tam_b1 - genGPC.b1 )^2 ))
itempars$corr_tam_b2 <- cor( genGPC.b2, tam_b2)
itempars$bias_tam_b2 <- mean( tam_b2 - genGPC.b2 )
itempars$RMSE_tam_b2 <- sqrt(mean( ( tam_b2 - genGPC.b2 )^2 ))
itempars$corr_tam_a <- cor( genGPC.a, tam_a)
itempars$bias_tam_a <- mean( tam_a - genGPC.a )
itempars$RMSE_tam_a <- sqrt( mean( ( tam_a - genGPC.a )^2 ))
# calculate corr, bias, RMSE for item parameters in ltm pacakge
itempars$corr_ltm_b1 <- cor( genGPC.b1, ltm_b1)
itempars$bias_ltm_b1 <- mean( ltm_b1 - genGPC.b1 )
itempars$RMSE_ltm_b1 <- sqrt(mean( ( ltm_b1 - genGPC.b1 )^2 ))
itempars$corr_ltm_b2 <- cor( genGPC.b2, ltm_b2)
itempars$bias_ltm_b2 <- mean( ltm_b2 - genGPC.b2 )
itempars$RMSE_ltm_b2 <- sqrt(mean( ( ltm_b2 - genGPC.b2 )^2 ))
itempars$corr_ltm_a <- cor( genGPC.a, ltm_a)
itempars$bias_ltm_a <- mean( ltm_a - genGPC.a )
itempars$RMSE_ltm_a <- sqrt( mean( ( ltm_a - genGPC.a )^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b1, bias_mirt_b1, RMSE_mirt_b1,
corr_mirt_b2, bias_mirt_b2, RMSE_mirt_b2,
corr_mirt_a, bias_mirt_a, RMSE_mirt_a) ~ N + I,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("Sample Size", "Test Length",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(mirt_recovery, 3)## Sample Size Test Length corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 0.982 -0.004 0.144 0.985 -0.003 0.150 0.798 0.011 0.109
## 2 1000 40 0.991 -0.008 0.100 0.993 -0.009 0.104 0.886 0.006 0.077
## 3 5000 40 0.998 -0.004 0.045 0.998 -0.003 0.047 0.974 0.000 0.034
## 4 500 60 0.983 -0.002 0.147 0.987 -0.006 0.151 0.790 0.011 0.109
## 5 1000 60 0.991 -0.010 0.104 0.993 -0.008 0.107 0.874 0.004 0.075
## 6 5000 60 0.998 -0.005 0.046 0.999 -0.003 0.048 0.971 0.000 0.034
## 7 500 80 0.983 -0.005 0.143 0.986 -0.006 0.152 0.808 0.008 0.106
## 8 1000 80 0.992 -0.008 0.101 0.994 -0.006 0.105 0.892 0.004 0.075
## 9 5000 80 0.998 -0.004 0.046 0.999 -0.002 0.047 0.974 -0.001 0.034
## 10 500 100 0.983 -0.005 0.146 0.986 -0.005 0.150 0.816 0.011 0.105
## 11 1000 100 0.991 -0.009 0.105 0.993 -0.006 0.108 0.895 0.004 0.074
## 12 5000 100 0.998 -0.006 0.047 0.999 -0.005 0.048 0.976 0.000 0.033
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b1, bias_tam_b1, RMSE_tam_b1,
corr_tam_b2, bias_tam_b2, RMSE_tam_b2,
corr_tam_a, bias_tam_a, RMSE_tam_a) ~ N + I,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("Sample Size", "Test Length",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(tam_recovery, 3)## Sample Size Test Length corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 0.982 -0.003 0.149 0.985 0.007 0.155 0.798 -0.001 0.107
## 2 1000 40 0.991 -0.012 0.103 0.993 -0.004 0.106 0.885 -0.006 0.076
## 3 5000 40 0.998 -0.006 0.048 0.998 0.004 0.049 0.974 -0.013 0.035
## 4 500 60 0.983 -0.015 0.172 0.987 0.013 0.176 0.788 -0.030 0.109
## 5 1000 60 0.991 -0.035 0.126 0.993 0.003 0.128 0.874 -0.043 0.084
## 6 5000 60 0.998 -0.023 0.066 0.999 0.016 0.069 0.971 -0.048 0.058
## 7 500 80 0.983 -0.033 0.191 0.986 0.029 0.201 0.806 -0.073 0.121
## 8 1000 80 0.992 -0.052 0.152 0.994 0.016 0.151 0.891 -0.081 0.105
## 9 5000 80 0.998 -0.037 0.097 0.999 0.034 0.105 0.974 -0.090 0.096
## 10 500 100 0.983 -0.041 0.214 0.986 0.055 0.221 0.814 -0.101 0.137
## 11 1000 100 0.991 -0.064 0.180 0.993 0.041 0.180 0.894 -0.111 0.129
## 12 5000 100 0.998 -0.053 0.125 0.999 0.049 0.132 0.975 -0.116 0.120
- Correlation, bias, and RMSE for item parameter recovery in
ltmpackage
ltm_recovery <- aggregate(cbind(corr_ltm_b1, bias_ltm_b1, RMSE_ltm_b1,
corr_ltm_b2, bias_ltm_b2, RMSE_ltm_b2,
corr_ltm_a, bias_ltm_a, RMSE_ltm_a) ~ N + I,
data=results, mean, na.rm=TRUE)
names(ltm_recovery) <- c("Sample Size", "Test Length",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(ltm_recovery, 3)## Sample Size Test Length corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 0.982 -0.014 0.159 0.985 0.012 0.164 0.798 -0.022 0.107
## 2 1000 40 0.991 -0.021 0.111 0.993 0.002 0.114 0.884 -0.026 0.078
## 3 5000 40 0.998 -0.015 0.056 0.998 0.012 0.058 0.974 -0.034 0.046
## 4 500 60 0.983 -0.036 0.190 0.987 0.017 0.194 0.787 -0.062 0.118
## 5 1000 60 0.991 -0.056 0.148 0.993 0.010 0.147 0.873 -0.077 0.103
## 6 5000 60 0.998 -0.039 0.092 0.999 0.029 0.098 0.970 -0.082 0.088
## 7 500 80 0.983 -0.060 0.220 0.986 0.037 0.231 0.805 -0.114 0.146
## 8 1000 80 0.992 -0.075 0.185 0.994 0.029 0.183 0.891 -0.122 0.138
## 9 5000 80 0.998 -0.057 0.133 0.999 0.048 0.144 0.974 -0.130 0.134
## 10 500 100 0.983 -0.115 0.266 0.986 0.025 0.257 0.814 -0.145 0.170
## 11 1000 100 0.991 -0.101 0.223 0.993 0.044 0.220 0.893 -0.151 0.164
## 12 5000 100 0.998 -0.080 0.171 0.999 0.066 0.179 0.975 -0.158 0.162