-
Notifications
You must be signed in to change notification settings - Fork 5
Multiple booklets GPC items
tmatta edited this page Oct 17, 2017
·
3 revisions
We examined item parameter recovery under the following conditions: 1 (IRT model) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 3 (test booklets).
- One IRT model was included: GPC model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value
- Two IRT R packages were evaluated:
TAM(version 2.4-9) andmirt(version 1.25) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration.
- Summary of item parameter recovery:
- TAM and mirt demonstrated a similar level of accuracy
- b1-parameter recovered well, with correlation ranging from 0.799 to 0.996, with bias ranging from -0.084 to 0.001, and with RMSE ranging from 0.07 to 0.876
- b2-parameter recovered well, with correlation ranging from 0.832 to 0.997, with bias ranging from -0.012 to 0.024, and with RMSE ranging from 0.072 to 0.69
- a-parameter recovered moderately, with correlation ranging from 0.262 to 0.939, with bias ranging from -0.013 to 0.23, and with RMSE ranging from 0.053 to 0.954
- For all parameters, sample sizes of 5000 consistently produced the most accurate results
- For b1- and b2-parameter, four levels of test lengths performed very similarly; when number of booklets increased, recovery accuracy decreased slightly
- For a-parameter, when sample size increased or when test length increased, recovery accuracy improved; but when number of booklets increased, recovery accuracy decreased
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a GPCM model
set.seed(4367) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_2pl = I,
thresholds = 2,
b_bounds = c(-2, 2),
a_bounds = c(.75, 1.25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
#------------------------------------------------------------------------------#
# Data simulation
#------------------------------------------------------------------------------#
set.seed(8088*(r+5))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment =
blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
b_par = item_pool$b,
a_par = item_pool$a,
d_par = list(item_pool$d1,
item_pool$d2))
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit GPCM model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = "gpcm", verbose = F,
technical = list( NCYCLES = 500))
# fit GPCM model using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml.2pl(resp, irtmodel = "GPCM",
control = list(maxiter = 200))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b1, b2, a in mirt package
mirt_b1 <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b1"]
mirt_b2 <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b2"]
mirt_a <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"a"]
# convert TAM output into GPCM parametrization
tam_b1 <- (tam.mod$item$AXsi_.Cat1/tam.mod$item$B.Cat1.Dim1)
tam_b2 <- (tam.mod$item$AXsi_.Cat2/tam.mod$item$B.Cat1.Dim1)
- (tam.mod$item$AXsi_.Cat1/tam.mod$item$B.Cat1.Dim1)
tam_a <- tam.mod$item$B.Cat1.Dim1
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow=1))
colnames(itempars) <- c("N", "I", "K", "rep")
# retrieve generated item parameters
genGPC.b1 <- item_pool$b + item_pool$d1
genGPC.b2 <- item_pool$b + item_pool$d2
genGPC.a <- item_pool$a
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b1 <- cor( genGPC.b1, mirt_b1)
itempars$bias_mirt_b1 <- mean( mirt_b1 - genGPC.b1 )
itempars$RMSE_mirt_b1 <- sqrt(mean( ( mirt_b1 - genGPC.b1 )^2 ))
itempars$corr_mirt_b2 <- cor( genGPC.b2, mirt_b2)
itempars$bias_mirt_b2 <- mean( mirt_b2 - genGPC.b2 )
itempars$RMSE_mirt_b2 <- sqrt(mean( ( mirt_b2 - genGPC.b2 )^2 ))
itempars$corr_mirt_a <- cor( genGPC.a, mirt_a)
itempars$bias_mirt_a <- mean( mirt_a - genGPC.a )
itempars$RMSE_mirt_a <- sqrt( mean( ( mirt_a - genGPC.a )^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b1 <- cor( genGPC.b1, tam_b1)
itempars$bias_tam_b1 <- mean( tam_b1 - genGPC.b1 )
itempars$RMSE_tam_b1 <- sqrt(mean( ( tam_b1 - genGPC.b1 )^2 ))
itempars$corr_tam_b2 <- cor( genGPC.b2, tam_b2)
itempars$bias_tam_b2 <- mean( tam_b2 - genGPC.b2 )
itempars$RMSE_tam_b2 <- sqrt(mean( ( tam_b2 - genGPC.b2 )^2 ))
itempars$corr_tam_a <- cor( genGPC.a, tam_a)
itempars$bias_tam_a <- mean( tam_a - genGPC.a )
itempars$RMSE_tam_a <- sqrt( mean( ( tam_a - genGPC.a )^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b1, bias_mirt_b1, RMSE_mirt_b1,
corr_mirt_b2, bias_mirt_b2, RMSE_mirt_b2,
corr_mirt_a, bias_mirt_a, RMSE_mirt_a) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("N", "I", "K",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(mirt_recovery, 3)## N I K corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 5 0.952 -0.005 0.234 0.961 -0.002 0.238 0.596 0.029 0.192
## 2 1000 40 5 0.976 -0.005 0.161 0.980 -0.004 0.168 0.725 0.012 0.131
## 3 5000 40 5 0.995 -0.001 0.071 0.996 -0.003 0.074 0.925 0.002 0.058
## 4 500 60 5 0.954 -0.003 0.238 0.966 -0.001 0.245 0.581 0.026 0.188
## 5 1000 60 5 0.979 -0.009 0.157 0.983 -0.007 0.170 0.727 0.014 0.127
## 6 5000 60 5 0.996 -0.002 0.071 0.997 -0.001 0.074 0.925 0.001 0.055
## 7 500 80 5 0.957 -0.008 0.228 0.965 -0.001 0.238 0.624 0.024 0.181
## 8 1000 80 5 0.978 -0.007 0.158 0.984 -0.007 0.161 0.759 0.014 0.125
## 9 5000 80 5 0.996 0.000 0.070 0.997 -0.001 0.072 0.933 0.003 0.054
## 10 500 100 5 0.956 -0.005 0.230 0.962 -0.001 0.240 0.634 0.026 0.178
## 11 1000 100 5 0.978 -0.007 0.161 0.981 -0.005 0.166 0.765 0.012 0.121
## 12 5000 100 5 0.995 -0.001 0.071 0.996 -0.001 0.072 0.939 0.002 0.053
## 13 500 40 10 0.894 -0.005 0.372 0.904 0.019 0.437 0.408 0.070 0.364
## 14 1000 40 10 0.950 -0.007 0.237 0.956 -0.011 0.255 0.521 0.036 0.227
## 15 5000 40 10 0.990 0.001 0.101 0.992 -0.002 0.107 0.828 0.007 0.095
## 16 500 60 10 0.905 -0.008 0.367 0.924 0.013 0.385 0.414 0.057 0.311
## 17 1000 60 10 0.954 -0.010 0.236 0.964 0.001 0.250 0.546 0.030 0.209
## 18 5000 60 10 0.991 -0.002 0.101 0.993 0.001 0.107 0.836 0.006 0.088
## 19 500 80 10 0.904 -0.008 0.356 0.926 -0.001 0.370 0.455 0.056 0.299
## 20 1000 80 10 0.953 -0.011 0.236 0.963 -0.003 0.241 0.591 0.030 0.197
## 21 5000 80 10 0.991 -0.003 0.102 0.993 -0.002 0.104 0.861 0.005 0.083
## 22 500 100 10 0.906 -0.012 0.358 0.919 -0.003 0.363 0.471 0.058 0.287
## 23 1000 100 10 0.955 -0.009 0.232 0.960 -0.003 0.243 0.609 0.024 0.188
## 24 5000 100 10 0.991 -0.005 0.100 0.992 0.000 0.103 0.871 0.004 0.079
## 25 500 40 15 0.799 -0.077 0.837 0.832 0.013 0.688 0.262 0.227 0.925
## 26 1000 40 15 0.916 -0.017 0.321 0.929 -0.003 0.332 0.409 0.072 0.373
## 27 5000 40 15 0.984 -0.002 0.130 0.986 -0.001 0.137 0.715 0.014 0.136
## 28 500 60 15 0.815 -0.010 0.697 0.840 0.024 0.680 0.283 0.123 0.525
## 29 1000 60 15 0.927 -0.010 0.303 0.939 0.005 0.338 0.437 0.057 0.288
## 30 5000 60 15 0.986 -0.004 0.126 0.989 0.000 0.134 0.755 0.010 0.118
## 31 500 80 15 0.851 -0.015 0.487 0.874 0.017 0.520 0.336 0.107 0.464
## 32 1000 80 15 0.927 -0.011 0.305 0.940 -0.001 0.324 0.484 0.048 0.274
## 33 5000 80 15 0.985 -0.004 0.127 0.988 0.000 0.132 0.785 0.009 0.111
## 34 500 100 15 0.850 -0.010 0.479 0.861 0.013 0.539 0.371 0.096 0.429
## 35 1000 100 15 0.929 -0.013 0.299 0.937 0.000 0.316 0.509 0.047 0.256
## 36 5000 100 15 0.985 -0.003 0.126 0.988 0.000 0.131 0.797 0.008 0.106
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b1, bias_tam_b1, RMSE_tam_b1,
corr_tam_b2, bias_tam_b2, RMSE_tam_b2,
corr_tam_a, bias_tam_a, RMSE_tam_a) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("N", "I", "K",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(tam_recovery, 3)## N I K corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 5 0.952 -0.005 0.233 0.961 -0.002 0.238 0.595 0.029 0.192
## 2 1000 40 5 0.976 -0.005 0.161 0.980 -0.004 0.168 0.725 0.012 0.131
## 3 5000 40 5 0.995 -0.001 0.071 0.996 -0.003 0.074 0.925 0.002 0.058
## 4 500 60 5 0.954 -0.005 0.239 0.966 -0.002 0.246 0.581 0.025 0.187
## 5 1000 60 5 0.979 -0.009 0.157 0.983 -0.006 0.170 0.727 0.013 0.126
## 6 5000 60 5 0.996 -0.002 0.071 0.997 0.000 0.074 0.925 0.000 0.055
## 7 500 80 5 0.957 -0.011 0.229 0.965 0.000 0.240 0.623 0.019 0.179
## 8 1000 80 5 0.978 -0.008 0.160 0.983 -0.004 0.162 0.759 0.008 0.123
## 9 5000 80 5 0.996 -0.004 0.070 0.997 -0.001 0.073 0.933 -0.003 0.054
## 10 500 100 5 0.956 -0.010 0.236 0.962 0.004 0.247 0.632 0.012 0.174
## 11 1000 100 5 0.978 -0.011 0.165 0.981 0.001 0.170 0.764 -0.001 0.119
## 12 5000 100 5 0.995 -0.007 0.074 0.996 0.004 0.075 0.938 -0.013 0.053
## 13 500 40 10 0.894 -0.005 0.372 0.904 0.019 0.437 0.407 0.070 0.366
## 14 1000 40 10 0.950 -0.007 0.237 0.956 -0.012 0.255 0.521 0.037 0.227
## 15 5000 40 10 0.990 0.001 0.101 0.992 -0.002 0.107 0.828 0.007 0.095
## 16 500 60 10 0.905 -0.008 0.367 0.924 0.013 0.385 0.414 0.057 0.311
## 17 1000 60 10 0.954 -0.010 0.236 0.964 0.001 0.250 0.546 0.031 0.209
## 18 5000 60 10 0.991 -0.002 0.102 0.993 0.001 0.107 0.836 0.006 0.088
## 19 500 80 10 0.904 -0.008 0.356 0.926 -0.002 0.370 0.454 0.056 0.299
## 20 1000 80 10 0.953 -0.011 0.236 0.963 -0.004 0.241 0.591 0.030 0.197
## 21 5000 80 10 0.991 -0.003 0.102 0.993 -0.002 0.105 0.861 0.005 0.083
## 22 500 100 10 0.906 -0.013 0.358 0.919 -0.004 0.363 0.470 0.059 0.288
## 23 1000 100 10 0.955 -0.009 0.233 0.960 -0.003 0.243 0.610 0.023 0.188
## 24 5000 100 10 0.991 -0.004 0.100 0.992 0.000 0.103 0.871 0.004 0.079
## 25 500 40 15 0.799 -0.084 0.876 0.832 0.011 0.690 0.265 0.230 0.954
## 26 1000 40 15 0.916 -0.017 0.321 0.929 -0.003 0.332 0.409 0.072 0.374
## 27 5000 40 15 0.984 -0.002 0.130 0.986 -0.001 0.137 0.715 0.014 0.137
## 28 500 60 15 0.815 -0.010 0.698 0.840 0.024 0.681 0.284 0.123 0.523
## 29 1000 60 15 0.927 -0.011 0.303 0.939 0.005 0.338 0.437 0.058 0.289
## 30 5000 60 15 0.986 -0.004 0.126 0.989 0.000 0.134 0.755 0.010 0.118
## 31 500 80 15 0.851 -0.015 0.487 0.874 0.017 0.520 0.336 0.107 0.466
## 32 1000 80 15 0.927 -0.011 0.306 0.940 -0.001 0.324 0.484 0.048 0.274
## 33 5000 80 15 0.985 -0.004 0.127 0.988 0.000 0.132 0.785 0.009 0.110
## 34 500 100 15 0.850 -0.010 0.479 0.861 0.013 0.539 0.367 0.099 0.448
## 35 1000 100 15 0.929 -0.013 0.299 0.937 0.000 0.316 0.509 0.048 0.256
## 36 5000 100 15 0.985 -0.003 0.126 0.988 0.000 0.131 0.797 0.008 0.106