Estimating ROC curves and corresponding AUC’s based on a gold standard
WinBUGS 1.4 code to accompany the paper entitled "Choi YK, Johnson WO, Collins MT, Gardner IA. Bayesian estimation of ROC curves in the absence of a gold standard. J Agric Biol Environ Stat. 2006 Jun; 11(2):210-229".
Code prepared by Young-Ku Choi, January 6, 2004
ykchoi@uic.edu
Institute for Health Research and Policy
University of Illinois at Chicago
Example from section 3.2
Binomial Sampling: Two tests / One population
Estimate parameters for binormal distributions, ROC curves using sensitivity and specificity, corresponding AUC’s, and the difference between AUC’s for two tests.
Data source:
Two ELISA tests, HerdChek® (IDEXX Laboratories Inc., Westbrook, Maine) and Parachek™ (Biocor Animal Health, Omaha, Nebraska), were performed to detect serum antibodies to Mycobacterium avium subsp. paratuberculosis, the cause of Johne’s disease in cattle. ELISA results were compared with fecal culture for the organism, as the gold standard. There were 88 cattle that were considered infected (fecal culture scores of at least 3) and 393 cattle that were considered non-infected (negative fecal cultures and from herds in which Johne’s disease was not suspected).
Model {
for (i in 1:88) {
S1[i] ~ dnorm(mu1[i],tau1[i]) # test 1
S2[i] ~ dnorm(condmu[i],condtau[i]) # test 2
mu1[i] <- lambda1[1]
tau1[i] <- gamma1[1]
condmu[i] <- lambda2[1]+rho[1]*sqrt(gamma1[1]/gamma2[1])*(S1[i]-lambda1[1])
condtau[i] <- (gamma2[1])/(1-pow(rho[1],2))
}
for (i in 89:481) {
S1[i] ~ dnorm(mu1[i],tau1[i]) # test 1
S2[i] ~ dnorm(condmu[i],condtau[i]) # test 2
mu1[i] <- lambda1[2]
tau1[i] <- gamma1[2]
condmu[i] <- lambda2[2]+rho[2]*sqrt(gamma1[2]/gamma2[2])*(S1[i]-lambda1[2])
condtau[i] <- (gamma2[2])/(1-pow(rho[2],2))
}
lambda1[1] ~ dnorm(0,0.001)I(lambda1[2],) # prior for the mean of disease group (test 1)
lambda2[1] ~ dnorm(0,0.001)I(lambda2[2],) # prior for the mean of disease group (test 2)
lambda1[2] ~ dnorm(0,0.001) # prior for the mean of non-disease group (test 1)
lambda2[2] ~ dnorm(0,0.001) # prior for the mean of non-disease group (test 2)
rho[1] ~ dunif(-1,1) # correlation coefficient for disease group
rho[2] ~ dunif(-1,1) # correlation coefficient for non-disease group
gamma1[1] ~ dgamma(0.001,0.001) # prior for the precision of disease group (test 1)
gamma2[1] ~ dgamma(0.001,0.001) # prior for the precision of disease group (test 2)
gamma1[2] ~ dgamma(0.001,0.001) # prior for precision of non-disease group (test 1)
gamma2[2] ~ dgamma(0.001,0.001) # prior for precision of non-disease group (test 2)
sigma1[1] <- 1/gamma1[1] # define the variance for disease group (test 1)
sigma1[2] <- 1/gamma1[2] # define the variance for non-disease group (test 1)
sigma2[1] <- 1/gamma2[1] # define the variance for disease group (test 2)
sigma2[2] <- 1/gamma2[2] # define the variance for non-disease group (test 2)
delta1 <- phi((-1.357-lambda1[1])/sqrt(sigma1[1]))
delta2 <- phi((-2.326-lambda2[1])/sqrt(sigma2[1]))
# AUC
AUC1 <- phi(-(lambda1[2]-lambda1[1])/sqrt(sigma1[2]+sigma1[1]))
AUC2 <- phi(-(lambda2[2]-lambda2[1])/sqrt(sigma2[2]+sigma2[1]))
diff <- AUC1-AUC2 # difference between two AUCs
# ROC curve
for(i in 1:111) {
c1[i] <- ((-8.1+0.1*i)-lambda1[1])/sqrt(sigma1[1]) # grid is from -3 to 8
se1[i] <- 1-phi(c1[i])
c2[i] <- ((-8.1+0.1*i)-lambda1[2])/sqrt(sigma1[2])
sp1[i] <- phi(c2[i])
c3[i] <- ((-8.1+0.1*i)-lambda2[1])/sqrt(sigma2[1])
se2[i] <- 1-phi(c3[i])
c4[i] <- ((-8.1+0.1*i)-lambda2[2])/sqrt(sigma2[2])
sp2[i] <- phi(c4[i])
}
}
list(lambda1=c(0,-2),lambda2=c(0,-2),gamma1=c(0.5,1),gamma2=c(0.5,1),rho=c(0,0))
list(
S1=c(-2.30,0.66,-0.45,-1.11,-1.11,0.52,0.93,-1.71,0.80,0.07,0.76,0.43,-2.66,0.46,
-0.34,0.53,0.17,-0.65,-0.04,0.17,0.35,-2.81,0.79,-1.17,0.40,0.13,0.88,0.83,
-2.30,-2.53,-1.35,-1.71,0.63,0.55,-3.22,0.13,0.07,0.60,-2.21,-3.51,-0.07,-1.56,
1.02,0.61,0.52,-3.51,0.56,0.03,0.60,0.46,-2.04,0.76,-1.39,0.51,0.15,-0.26,
0.98,0.80,0.24,0.12,0.67,-0.29,0.18,-0.34,-0.07,-0.31,0.41,-1.05,-1.35,0.39,
0.25,-1.71,0.92,-1.20,-2.30,-1.27,0.49,-0.26,-2.81,-2.81,-0.82,-0.73,-0.20,0.59,
0.58,-2.41,-1.77,-2.04,
-3.51,-2.66,-2.30,-1.77,-3.00,-3.22,-2.30,-2.21,-2.53,-3.22,-2.21,-3.22,-3.22,-3.22,
-3.00,-3.22,-2.41,-2.21,-3.51,-2.12,-3.91,-2.04,-2.53,-2.81,-1.56,-1.77,-1.90,-2.81,
-1.90,-1.05,-3.22,-1.90,-2.41,-3.51,-2.81,-2.41,-3.00,-2.66,-3.22,-1.27,-2.66,-1.66,
-1.56,-2.81,-2.04,-3.00,-3.91,-4.61,-1.90,-2.04,-2.21,-2.21,-1.61,-1.24,-1.90,-3.91,
-3.00,-2.81,-2.12,-2.12,-3.51,-2.12,-2.21,-2.81,-2.66,-4.61,-2.12,-2.21,-3.91,-3.51,
-1.71,-2.81,-3.91,-4.61,-2.53,-3.22,-3.00,-2.53,-3.00,-4.61,-2.66,-4.61,-1.90,-2.53,
-2.81,-2.66,-2.66,-2.53,-2.81,-3.00,-2.53,-3.91,-1.56,-3.91,-2.53,-2.81,-3.51,-3.22,
-2.53,-3.22,-2.53,-1.71,-4.61,-3.22,-3.22,-3.51,-2.53,-2.81,-2.66,-3.00,-2.30,-1.83,
-2.66,-3.22,-2.81,-4.61,-2.30,-2.53,-2.81,-3.91,-4.61,-1.71,-2.53,-3.00,-3.51,-2.66,
-1.39,-2.30,-3.51,-2.81,-2.41,-3.00,-2.81,-2.53,-2.53,-3.91,-1.11,-2.41,0.17,-1.24,
-2.04,-2.04,-1.20,-3.51,-2.41,-2.12,-3.22,-0.42,-1.11,-1.83,-2.21,-2.66,-3.51,-3.22,
-2.12,-1.77,-0.54,-2.30,-2.12,-2.30,-4.61,-2.66,-1.71,-1.97,-1.51,-1.61,-2.41,-2.53,
-2.66,-3.91,-1.61,-3.00,-2.41,-2.04,-0.99,-2.12,-3.00,-1.97,-1.56,-3.51,-2.81,-2.41,
-1.77,-2.53,-2.12,-2.66,-3.51,-4.61,-3.22,-3.22,-4.61,-3.22,-1.71,-2.41,-2.04,-3.00,
-3.22,-3.51,-2.53,-3.91,-2.30,-2.04,-3.00,-2.30,-3.51,-2.81,-0.97,-2.30,-2.66,-3.91,
-2.66,-3.22,-3.00,-3.00,-2.81,-3.91,-3.00,-3.22,-3.22,-3.91,-3.51,-2.81,-3.00,-2.81,
-3.51,-1.83,-4.61,-3.51,-2.53,-3.22,-2.12,-2.81,-2.66,-1.47,-2.81,-2.41,-1.77,-3.22,
-2.81,-2.41,-2.41,-2.41,-1.35,-3.00,-3.22,-3.51,-2.53,-3.22,-2.53,-3.22,-2.41,-2.04,
-4.61,-2.53,-2.30,-2.66,-2.41,-2.66,-3.22,-0.97,-3.51,-3.51,-3.51,-3.22,-2.30,-1.83,
-2.81,-3.22,-3.22,-1.39,-3.00,-2.53,-2.53,-2.04,-3.51,-2.66,-2.66,-2.66,-3.22,-3.00,
-2.41,-2.53,-2.41,-1.08,-2.81,-3.22,-1.66,-4.61,-3.91,-3.51,-3.91,-4.61,-2.41,-3.00,
-3.91,-2.66,-3.00,-3.00,-2.81,-3.91,-2.66,-3.00,-3.51,-2.66,-3.91,-3.22,-1.27,-2.81,
-1.31,-3.51,-2.21,-2.41,-2.04,-2.81,-2.81,-4.61,-3.51,-1.97,-4.61,-2.53,-3.00,-1.71,
-3.91,-4.61,-3.22,-3.51,-0.60,-3.91,-1.39,-3.22,-2.41,-1.61,-1.90,-3.22,-3.22,-2.81,
-2.30,-2.04,-1.83,-2.21,-1.51,-2.21,-2.21,-2.66,-2.21,-2.66,-2.66,-3.22,-2.66,-1.83,
-2.66,-1.83,-3.22,-3.91,-2.21,-3.22,-3.22,-2.66,-2.66,-2.04,-2.41,-1.61,-3.91,-2.66,
-2.21,-2.41,-1.71,-3.22,-1.83,-2.41,-2.66,-2.66,-2.04,-1.83,-3.22,-1.83,-3.22,-2.21,
-2.41,-2.41,-2.66,-2.41,-3.91,-3.22,-2.41,-1.83,-2.04,-2.21,-2.66,-2.04,-3.22,-3.91,
-2.66),
S2=c(-1.09,-0.69,-1.26,-0.54,-2.09,-0.76,0.13,-1.86,-0.21,-1.48,-0.02,-0.17,-1.19,-1.51,
-0.92,0.21,-1.01,-1.97,-1.02,-0.06,-1.26,-2.34,-0.29,-2.44,-1.03,-1.97,-0.22,-1.45,
-2.53,-2.03,-1.90,-2.27,0.81,0.17,-1.45,-0.56,-0.32,-0.02,-2.03,-2.21,-1.29,-1.31,
0.22,0.17,0.37,-3.06,-0.33,-0.97,0.84,-0.03,-2.66,0.01,-1.39,0.81,-0.45,-0.87,
0.21,0.46,-0.53,-0.29,0.51,-0.56,-0.67,-0.54,-0.51,-0.99,-0.21,-2.04,-1.97,-0.16,
0.43,-1.77,0.19,-1.08,-2.30,-1.66,-0.78,-0.69,-2.41,-2.41,-1.51,-2.53,-0.33,0.20,
-0.24,-2.53,-0.80,-2.12,
-2.98,-2.90,-2.66,-2.75,-2.83,-2.83,-2.92,-2.98,-2.88,-2.55,-2.78,-2.80,-2.94,-2.98,
-3.02,-2.83,-2.86,-2.98,-2.88,-2.63,-2.94,-2.48,-2.86,-2.88,-2.78,-2.98,-2.80,-2.94,
-2.60,-2.63,-2.92,-2.81,-2.62,-2.86,-2.83,-2.81,-2.78,-2.76,-2.63,-1.98,-2.81,-2.28,
-2.69,-2.60,-2.31,-2.63,-2.70,-2.88,-2.55,-2.60,-2.45,-2.73,-2.78,-1.81,-2.69,-2.51,
-2.72,-2.67,-1.53,-2.78,-2.65,-2.17,-2.36,-2.75,-2.49,-2.50,-2.59,-2.69,-2.20,-2.40,
-2.23,-2.12,-2.65,-2.44,-2.67,-2.55,-2.56,-2.23,-2.70,-2.24,-2.44,-1.93,-2.28,-2.72,
-2.25,-2.44,-2.66,-2.65,-2.70,-2.55,-2.30,-2.67,-2.73,-2.40,-3.06,-2.92,-3.06,-3.06,
-3.02,-2.86,-2.94,-3.00,-2.88,-2.88,-3.08,-2.92,-3.06,-2.90,-2.92,-3.04,-3.10,-2.96,
-3.02,-3.08,-2.88,-2.92,-2.96,-3.08,-2.94,-3.00,-3.04,-2.67,-2.98,-3.02,-3.00,-3.04,
-2.44,-3.08,-3.02,-2.98,-2.94,-2.90,-2.47,-3.06,-3.06,-3.04,-1.87,-2.14,-2.45,-2.62,
-2.63,-2.19,-2.54,-2.67,-2.54,-2.27,-2.73,-2.50,-2.60,-2.62,-2.62,-2.40,-2.59,-2.35,
-2.44,-2.48,-2.58,-2.55,-2.75,-2.73,-2.80,-2.49,-2.51,-2.66,-2.60,-2.60,-2.67,-2.43,
-2.73,-2.59,-2.67,-2.49,-2.54,-2.23,-2.49,-2.11,-2.59,-2.41,-2.51,-2.75,-2.81,-2.38,
-2.47,-2.45,-2.75,-2.96,-2.83,-2.98,-2.90,-2.90,-2.92,-2.98,-2.92,-2.60,-2.98,-2.96,
-2.92,-2.94,-2.81,-3.00,-2.54,-2.67,-2.96,-2.83,-2.90,-2.02,-2.70,-2.81,-1.97,-2.69,
-2.30,-2.80,-2.90,-2.67,-2.86,-2.88,-2.75,-2.65,-2.80,-2.73,-2.76,-2.73,-2.78,-2.76,
-2.83,-2.53,-2.85,-2.78,-2.85,-3.08,-3.10,-3.08,-3.08,-3.02,-3.08,-2.80,-2.98,-3.04,
-3.02,-3.04,-3.08,-3.02,-2.86,-2.98,-3.00,-3.08,-3.08,-2.98,-3.06,-3.04,-3.06,-3.02,
-3.02,-3.08,-2.83,-3.08,-3.08,-2.90,-2.98,-3.06,-2.94,-2.98,-3.06,-2.85,-3.02,-3.00,
-3.06,-3.06,-3.08,-2.92,-3.10,-3.04,-3.08,-2.94,-3.10,-3.00,-2.98,-3.06,-2.98,-3.06,
-3.04,-3.08,-2.65,-2.85,-3.00,-3.04,-2.51,-2.85,-2.63,-2.90,-2.56,-2.67,-2.78,-2.65,
-2.67,-2.76,-2.78,-2.50,-2.32,-2.54,-2.56,-2.47,-2.73,-2.44,-2.76,-2.76,-2.54,-2.62,
-2.54,-2.83,-2.65,-2.55,-2.59,-2.75,-2.66,-2.86,-2.69,-2.75,-2.75,-2.72,-2.15,-2.63,
-2.80,-2.63,-2.48,-2.47,-2.34,-2.63,-2.50,-2.36,-2.75,-2.51,-2.30,-2.70,-2.69,-2.65,
-2.40,-2.81,-3.00,-2.83,-2.81,-2.90,-2.86,-2.81,-2.25,-2.88,-3.00,-2.90,-2.81,-2.88,
-2.80,-2.85,-2.98,-2.80,-2.83,-2.63,-2.58,-2.85,-2.76,-2.75,-2.85,-2.86,-2.81,-2.78,
-2.75,-2.69,-2.62,-2.81,-2.92,-2.88,-2.92,-2.85,-2.94,-2.86,-2.96,-2.67,-2.63,-2.83,
-2.81,-2.70,-2.69,-2.90,-2.80,-2.85,-2.67,-2.43,-2.85,-2.86,-2.98,-2.86,-2.94,-2.94,
-2.83))
node |
mean |
sd |
MC error |
2.50% |
median |
97.50% |
start |
sample |
AUC1 |
0.9318 |
0.0162 |
3.236E-4 |
0.8959 |
0.9334 |
0.9593 |
501 |
9500 |
AUC2 |
0.9625 |
0.01316 |
2.488E-4 |
0.9318 |
0.9641 |
0.9833 |
501 |
9500 |
delta1 |
0.238 |
0.03691 |
8.022E-4 |
0.1693 |
0.2366 |
0.3152 |
501 |
9500 |
delta2 |
0.07813 |
0.02163 |
4.244E-4 |
0.04171 |
0.07615 |
0.1268 |
501 |
9500 |
diff |
-0.03064 |
0.01228 |
1.343E-4 |
-0.05641 |
-0.03022 |
-0.007731 |
501 |
9500 |
lambda1[1] |
-0.4688
|
0.1338 |
0.00308 |
-0.7292 |
-0.469 |
-0.2062 |
501 |
9500 |
lambda1[2] |
-2.699 |
0.04106 |
4.094E-4 |
-2.78 |
-2.699 |
-2.619 |
501 |
9500 |
lambda2[1] |
-0.9385 |
0.1041 |
0.002333 |
-1.14 |
-0.9393 |
-0.7317 |
501 |
9500 |
lambda2[2] |
-2.744 |
0.01289 |
1.429E-4 |
-2.77 |
-2.744 |
-2.719 |
501 |
9500 |
rho[1] |
0.7868 |
0.04121 |
6.812E-4 |
0.6965 |
0.7905 |
0.8571 |
501 |
9500 |
rho[2] |
0.1921 |
0.04841 |
4.682E-4 |
0.09728 |
0.1932 |
0.2848 |
501 |
9500 |
sigma1[1] |
1.557 |
0.2398 |
0.00395 |
1.159 |
1.533 |
2.102 |
501 |
9500 |
sigma1[2] |
0.6712 |
0.04826 |
4.539E-4 |
0.5829 |
0.6687 |
0.771 |
501 |
9500 |
sigma2[1] |
0.9526 |
0.1441 |
0.002362 |
0.7113 |
0.9383 |
1.273 |
501 |
9500 |
sigma2[2] |
0.06541 |
0.004792 |
4.722E-5 |
0.05666 |
0.06516 |
0.0754 |
501 |
9500 |