-### EXAMPLE 2_b: demonstrates how measurement error can lead to a type sign error in a covariate
-### This is the same as example 2, only instead of x->k we have k->x.
-### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
-### Even when you include the proxy variable in the regression.
-### But with some ground truth and multiple imputation, you can fix it.
+### EXAMPLE 2_b: demonstrates how measurement error can lead to a type
+### sign error in a covariate This is the same as example 2, only
+### instead of x->k we have k->x. Even when you have a good
+### predictor, if it's biased against a covariate you can get the
+### wrong sign. Even when you include the proxy variable in the
+### regression. But with some ground truth and multiple imputation,
+### you can fix it.
library(argparser)
library(mecor)
library(arrow)
library(Amelia)
library(Zelig)
+
library(predictionError)
-options(amelia.parallel="no",
- amelia.ncpus=1)
+options(amelia.parallel="no", amelia.ncpus=1)
source("simulation_base.R")
#### how much power do we get from the model in the first place? (sweeping N and m)
####
-simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_variance=0.025, gx_explained_variance=0.15, prediction_accuracy=0.73, seed=1){
+simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, seed=1){
set.seed(seed)
- g <- rbinom(N, 1, 0.5)
-
- x.var.epsilon <- var(Bgx *g) * ((1-gx_explained_variance)/gx_explained_variance)
- x.epsilon <- rnorm(N,sd=sqrt(x.var.epsilon))
- xprime <- Bgx * g + x.epsilon
- x <- as.integer(logistic(scale(xprime)) > 0.5)
+ z <- rbinom(N, 1, 0.5)
+ # x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
+ xprime <- Bzx * z #+ x.var.epsilon
+ x <- rbinom(N,1,plogis(xprime))
- y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bxy*x,Bgy*g)) * ((1-y_explained_variance)/y_explained_variance)
+ y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)
y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
- y <- Bgy * g + Bxy * x + y.epsilon
+ y <- Bzy * z + Bxy * x + y.epsilon
- df <- data.table(x=x,xprime=xprime,y=y,g=g)
+ df <- data.table(x=x,y=y,z=z)
if(m < N){
df <- df[sample(nrow(df), m), x.obs := x]
df <- df[, x.obs := x]
}
- df <- df[,w_pred:=x]
- df <- df[sample(1:N,(1-prediction_accuracy)*N),w_pred:=(w_pred-1)**2]
- w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
- df <- df[,':='(w=w, w_pred = w_pred)]
+ ## how can you make a model with a specific accuracy?
+ w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy
+
+ ## how can you make a model with a specific accuracy, with a continuous latent variable.
+ # now it makes the same amount of mistake to each point, probably
+ # add mean0 noise to the odds.
+
+ w.noisey.odds = rlogis(N,qlogis(w0))
+ df[,w := plogis(w.noisey.odds)]
+ df[,w_pred:=as.integer(w > 0.5)]
+ (mean(df$x==df$w_pred))
return(df)
}
parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=500, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=100, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=57, help='seed for the rng')
parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')
-parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
-parser <- add_argument(parser, "--gx_explained_variance", help='what proportion of the variance of x can be explained by g?', default=0.15)
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
+# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
-
+parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1)
args <- parse_args(parser)
B0 <- 0
-Bxy <- 0.2
-Bgy <- -0.2
-Bgx <- 0.4
+Bxy <- 0.3
+Bzy <- -0.3
+Bzx <- args$Bzx
-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, Bgx, seed=args$seed, y_explained_variance = args$y_explained_variance, gx_explained_variance = args$gx_explained_variance, prediction_accuracy=args$prediction_accuracy)
+if (args$m < args$N){
-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'Bgx'=Bgx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'gx_explained_variance' = args$gx_explained_variance, "prediction_accuracy"=args$prediction_accuracy)
-outline <- run_simulation(df, result)
+ df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance, prediction_accuracy=args$prediction_accuracy)
-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
- logdata <- read_feather(args$outfile)
- logdata <- rbind(logdata,as.data.table(outline))
-} else {
- logdata <- as.data.table(outline)
-}
+ result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="")
-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)
+ outline <- run_simulation(df, result)
+
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+ if(file.exists(args$outfile)){
+ logdata <- read_feather(args$outfile)
+ logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+ } else {
+ logdata <- as.data.table(outline)
+ }
+
+ print(outline)
+ write_feather(logdata, args$outfile)
+ unlock(outfile_lock)
+}
## one way to do it is by adding correlation to x.obs and y that isn't in w.
## in other words, the model is missing an important feature of x.obs that's related to y.
-simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
set.seed(seed)
# make w and y dependent
- g <- rbinom(N, 1, 0.5)
- x <- rbinom(N, 1, 0.5)
+ z <- rbinom(N, 1, 0.5)
+ x <- rbinom(N, 1, Bzx * z + 0.5)
- y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bgy*g,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
+ y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
- y <- Bgy * g + Bxy * x + y.epsilon
-
- df <- data.table(x=x,y=y,g=g)
+ y <- Bzy * z + Bxy * x + y.epsilon
+
+ df <- data.table(x=x,y=y,z=z)
if(m < N){
df <- df[sample(nrow(df), m), x.obs := x]
df <- df[, x.obs := x]
}
- df <- df[,w_pred:=x]
-
- pg <- mean(g)
- px <- mean(x)
- accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
+ ## px <- mean(x)
+ ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
- # this works because of conditional probability
- accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
- accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
+ ## # this works because of conditional probability
+ ## accuracy_x0 <- prediction_accuracy / (px*(accuracy_imbalance_ratio) + (1-px))
+ ## accuracy_x1 <- accuracy_imbalance_ratio * accuracy_x0
- dfg0 <- df[g==0]
- ng0 <- nrow(dfg0)
- dfg1 <- df[g==1]
- ng1 <- nrow(dfg1)
+ ## x0 <- df[x==0]$x
+ ## x1 <- df[x==1]$x
+ ## nx1 <- nrow(df[x==1])
+ ## nx0 <- nrow(df[x==0])
- dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
- dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
+ ## yx0 <- df[x==0]$y
+ ## yx1 <- df[x==1]$y
+
+ # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+ ## acc.x0 <- plogis(0.5*scale(yx0) + qlogis(accuracy_x0))
+ ## acc.x1 <- plogis(1.5*scale(yx1) + qlogis(accuracy_x1))
- df <- rbind(dfg0,dfg1)
+ ## w0x0 <- (1-x0)**2 + (-1)**(1-x0) * acc.x0
+ ## w0x1 <- (1-x1)**2 + (-1)**(1-x1) * acc.x1
+ pz <- mean(z)
+ accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
- w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
- df <- df[,':='(w=w, w_pred = w_pred)]
+ # this works because of conditional probability
+ accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+ accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
+
+ z0x0 <- df[(z==0) & (x==0)]$x
+ z0x1 <- df[(z==0) & (x==1)]$x
+ z1x0 <- df[(z==1) & (x==0)]$x
+ z1x1 <- df[(z==1) & (x==1)]$x
+
+ yz0x0 <- df[(z==0) & (x==0)]$y
+ yz0x1 <- df[(z==0) & (x==1)]$y
+ yz1x0 <- df[(z==1) & (x==0)]$y
+ yz1x1 <- df[(z==1) & (x==1)]$y
+
+ nz0x0 <- nrow(df[(z==0) & (x==0)])
+ nz0x1 <- nrow(df[(z==0) & (x==1)])
+ nz1x0 <- nrow(df[(z==1) & (x==0)])
+ nz1x1 <- nrow(df[(z==1) & (x==1)])
+
+ yz1 <- df[z==1]$y
+ yz1 <- df[z==1]$y
+
+ # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+ acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
+ acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+ acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+ acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
+
+ w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
+ w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
+ w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
+ w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
+
+ ##perrorz0 <- w0z0*(pyz0)
+ ##perrorz1 <- w0z1*(pyz1)
+
+ w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
+ w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
+ w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
+ w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
+
+ df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
+ df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]
+ df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]
+ df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]
+
+ df[,w_pred:=as.integer(w > 0.5)]
+ print(mean(df[z==0]$x == df[z==0]$w_pred))
+ print(mean(df[z==1]$x == df[z==1]$w_pred))
+ print(mean(df$w_pred == df$x))
return(df)
}
parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=432, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1400, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=50, help='seed for the rng')
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+
args <- parse_args(parser)
B0 <- 0
-Bxy <- 0.2
-Bgy <- -0.2
-
-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
+Bxy <- 0.3
+Bzy <- args$Bzy
-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
+if(args$m < args$N){
+ df <- simulate_data(args$N, args$m, B0, Bxy, args$Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
-outline <- run_simulation_depvar(df=df, result)
+ result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, error='')
+ outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x+z+y+x:y, truth_formula=x~z)
+
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+ if(file.exists(args$outfile)){
+ logdata <- read_feather(args$outfile)
+ logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
+ } else {
+ logdata <- as.data.table(outline)
+ }
-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
- logdata <- read_feather(args$outfile)
- logdata <- rbind(logdata,as.data.table(outline))
-} else {
- logdata <- as.data.table(outline)
+ print(outline)
+ write_feather(logdata, args$outfile)
+ unlock(outfile_lock)
}
-
-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)
## one way to do it is by adding correlation to x.obs and y that isn't in w.
## in other words, the model is missing an important feature of x.obs that's related to y.
-simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
set.seed(seed)
# make w and y dependent
- g <- rbinom(N, 1, 0.5)
+ z <- rbinom(N, 1, 0.5)
x <- rbinom(N, 1, 0.5)
- ystar <- Bgy * g + Bxy * x
- y <- rbinom(N,1,logistic(ystar))
+ ystar <- Bzy * z + Bxy * x
+ y <- rbinom(N,1,plogis(ystar))
- # glm(y ~ x + g, family="binomial")
+ # glm(y ~ x + z, family="binomial")
- df <- data.table(x=x,y=y,ystar=ystar,g=g)
+ df <- data.table(x=x,y=y,ystar=ystar,z=z)
if(m < N){
df <- df[sample(nrow(df), m), y.obs := y]
df <- df[,w_pred:=y]
- pg <- mean(g)
+ pz <- mean(z)
accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
# this works because of conditional probability
- accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
- accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
+ accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+ accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
- dfg0 <- df[g==0]
- ng0 <- nrow(dfg0)
- dfg1 <- df[g==1]
- ng1 <- nrow(dfg1)
- dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
- dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
+ yz0 <- df[z==0]$y
+ yz1 <- df[z==1]$y
+ nz1 <- nrow(df[z==1])
+ nz0 <- nrow(df[z==0])
- df <- rbind(dfg0,dfg1)
+ acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0))
+ acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1))
+
+ w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0
+ w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1
+
+ w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0))
+ w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1))
+ df[z==0,w:=plogis(w0z0.noisy.odds)]
+ df[z==1,w:=plogis(w0z1.noisy.odds)]
- wmod <- glm(y.obs ~ w_pred,data=df[!is.null(y.obs)],family=binomial(link='logit'))
- w <- predict(wmod,df,type='response')
+ df[,w_pred:=as.integer(w > 0.5)]
- df <- df[,':='(w=w)]
+ print(mean(df[y==0]$y == df[y==0]$w_pred))
+ print(mean(df[y==1]$y == df[y==1]$w_pred))
+ print(mean(df$w_pred == df$y))
return(df)
}
parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
args <- parse_args(parser)
B0 <- 0
-Bxy <- 0.2
-Bgy <- -0.2
+Bxy <- 0.7
+Bzy <- -0.7
-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
+if(args$m < args$N){
+ df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
+ result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
-outline <- run_simulation_depvar(df=df, result)
+ outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x)
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
- logdata <- read_feather(args$outfile)
- logdata <- rbind(logdata,as.data.table(outline))
-} else {
- logdata <- as.data.table(outline)
-}
+ if(file.exists(args$outfile)){
+ logdata <- read_feather(args$outfile)
+ logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+ } else {
+ logdata <- as.data.table(outline)
+ }
-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)
+ print(outline)
+ write_feather(logdata, args$outfile)
+ unlock(outfile_lock)
+}
SHELL=bash
-Ns=[500,1000,10000]
-ms=[50, 100, 250, 500]
+Ns=[1000,3600,14400]
+ms=[75,150,300]
seeds=[$(shell seq -s, 1 250)]
+explained_variances=[0.1]
+
all:remembr.RDS
-srun=srun -A comdata -p compute-bigmem --time=10:00:00 --mem 4G -c 1
+srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1
+
+
+joblists:example_1_jobs example_2_jobs example_3_jobs
+
+# test_true_z_jobs: test_true_z.R simulation_base.R
+# grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
-example_1_jobs: 01_two_covariates.R
- grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"]}' --outfile example_1_jobs
+# test_true_z.feather: test_true_z_jobs
+# rm -f test_true_z.feather
+# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 test_true_z_jobs
+# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs
-example_1.feather: example_1_jobs
+
+example_1_jobs: 01_two_covariates.R simulation_base.R
+ grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs
+
+example_1.feather: example_1_jobs
rm -f example_1.feather
- sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_1_jobs
- sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
+ sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
+# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
-example_2_jobs: example_2.R
- grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"]}' --outfile example_2_jobs
+example_2_jobs: 02_indep_differential.R simulation_base.R
+ grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs
-example_2.feather: example_2_jobs
+example_2.feather: example_2_jobs
rm -f example_2.feather
- sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_jobs
- sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
+ sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
+# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
# example_2_B_jobs: example_2_B.R
# grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
# rm -f example_2_B.feather
# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
-example_3_jobs: 03_depvar_differential.R
- grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"]}' --outfile example_3_jobs
+example_3_jobs: 03_depvar_differential.R simulation_base.R
+ grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
example_3.feather: example_3_jobs
- rm -f example_3.feather
- sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_3_jobs
- sbatch --wait --verbose --array=3001-6000 run_simulation.sbatch 0 example_3_jobs
+ rm -f example_3.feather
+ sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
-remembr.RDS:example_1.feather example_2.feather example_3.feather
+
+remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R
+ rm -f remembr.RDS
${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
clean:
rm *.feather
- rm remembr.RDS
-
+ rm -f remembr.RDS
+ rm -f example_*_jobs
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
# example_2_B_mecor_jobs:
--- /dev/null
+library(formula.tools)
+library(matrixStats)
+
+## df: dataframe to model
+## outcome_formula: formula for y | x, z
+## outcome_family: family for y | x, z
+## proxy_formula: formula for w | x, z, y
+## proxy_family: family for w | x, z, y
+## truth_formula: formula for x | z
+## truth_family: family for x | z
+
+### ideal formulas for example 1
+# test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
+
+### ideal formulas for example 2
+# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
+
+
+## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
+
+ nll <- function(params){
+ df.obs <- model.frame(outcome_formula, df)
+ proxy.variable <- all.vars(proxy_formula)[1]
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ response.var <- all.vars(outcome_formula)[1]
+ y.obs <- with(df.obs,eval(parse(text=response.var)))
+ outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
+
+ param.idx <- 1
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+ outcome.params <- params[param.idx:n.outcome.model.covars]
+ param.idx <- param.idx + n.outcome.model.covars
+
+ if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
+ ll.y.obs <- vector(mode='numeric', length=length(y.obs))
+ ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
+ ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
+ }
+
+ df.obs <- model.frame(proxy_formula,df)
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+ proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+
+ param.idx <- param.idx + n.proxy.model.covars
+ proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
+
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+ ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
+ ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
+ ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ ll.obs <- sum(ll.y.obs + ll.w.obs)
+
+ df.unobs <- df[is.na(df[[response.var]])]
+ df.unobs.y1 <- copy(df.unobs)
+ df.unobs.y1[[response.var]] <- 1
+ df.unobs.y0 <- copy(df.unobs)
+ df.unobs.y0[[response.var]] <- 1
+
+ ## integrate out y
+ outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
+
+ if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
+ ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
+ ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
+ ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
+ ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
+ }
+
+ proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
+ proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
+ proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
+
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+ ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
+ ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
+ ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
+ ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+
+ ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
+ ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
+ ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
+ ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
+ ll <- ll.unobs + ll.obs
+ return(-ll)
+ }
+
+ params <- colnames(model.matrix(outcome_formula,df))
+ lower <- rep(-Inf, length(params))
+ proxy.params <- colnames(model.matrix(proxy_formula, df))
+ params <- c(params, paste0('proxy_',proxy.params))
+ lower <- c(lower, rep(-Inf, length(proxy.params)))
+ start <- rep(0.1,length(params))
+ names(start) <- params
+
+ fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ return(fit)
+}
+
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
+
+ measrr_mle_nll <- function(params){
+ df.obs <- model.frame(outcome_formula, df)
+
+ proxy.variable <- all.vars(proxy_formula)[1]
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+
+ response.var <- all.vars(outcome_formula)[1]
+ y.obs <- with(df.obs,eval(parse(text=response.var)))
+
+ outcome.model.matrix <- model.matrix(outcome_formula, df)
+
+ param.idx <- 1
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+ outcome.params <- params[param.idx:n.outcome.model.covars]
+ param.idx <- param.idx + n.outcome.model.covars
+
+ ## likelihood for the fully observed data
+ if(outcome_family$family == "gaussian"){
+ sigma.y <- params[param.idx]
+ param.idx <- param.idx + 1
+ ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
+ }
+
+ df.obs <- model.frame(proxy_formula,df)
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+ proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+ param.idx <- param.idx + n.proxy.model.covars
+ proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
+
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+ ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
+ ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
+ ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ df.obs <- model.frame(truth_formula, df)
+ truth.variable <- all.vars(truth_formula)[1]
+ truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
+ truth.model.matrix <- model.matrix(truth_formula,df)
+ n.truth.model.covars <- dim(truth.model.matrix)[2]
+
+ truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+ if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
+ ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
+ ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
+ ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
+
+ ## likelihood for the predicted data
+ ## integrate out the "truth" variable.
+
+ if(truth_family$family=='binomial'){
+ df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
+ df.unobs.x1 <- copy(df.unobs)
+ df.unobs.x1[,'x'] <- 1
+ df.unobs.x0 <- copy(df.unobs)
+ df.unobs.x0[,'x'] <- 0
+ outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
+
+ outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
+ outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
+ if(outcome_family$family=="gaussian"){
+ ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+ ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+ }
+
+ if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
+
+ proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+ proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+ proxy.unobs <- df.unobs[[proxy.variable]]
+ ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
+ ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
+
+ ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
+ ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
+
+ ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
+ ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
+ }
+
+ if(truth_family$link=='logit'){
+ truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
+ ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
+ ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
+ }
+ }
+
+ ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
+ ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
+ ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
+ return(-(ll.unobs + ll.obs))
+ }
+
+ outcome.params <- colnames(model.matrix(outcome_formula,df))
+ lower <- rep(-Inf, length(outcome.params))
+
+ if(outcome_family$family=='gaussian'){
+ params <- c(outcome.params, 'sigma_y')
+ lower <- c(lower, 0.00001)
+ } else {
+ params <- outcome.params
+ }
+
+ proxy.params <- colnames(model.matrix(proxy_formula, df))
+ params <- c(params, paste0('proxy_',proxy.params))
+ lower <- c(lower, rep(-Inf, length(proxy.params)))
+
+ truth.params <- colnames(model.matrix(truth_formula, df))
+ params <- c(params, paste0('truth_', truth.params))
+ lower <- c(lower, rep(-Inf, length(truth.params)))
+ start <- rep(0.1,length(params))
+ names(start) <- params
+
+ fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+
+ return(fit)
+}
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
args <- parse_args(parser)
-build_plot_dataset <- function(df){
- x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
- x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
- zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
- bias = Bxy - Bxy.est.naive,
- Bxy.est.naive = Bxy.est.naive,
- sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
-
- x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bxy.est.naive),
- var.est = var(Bxy.est.naive),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='x',
- method='Naive'
- ),
- by=c('N','m')]
-
- g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
- g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
- zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
- bias = Bgy - Bgy.est.naive,
- Bgy.est.naive = Bgy.est.naive,
- sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
-
- g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bgy.est.naive),
- var.est = var(Bgy.est.naive),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='g',
- method='Naive'
- ),
- by=c('N','m')]
-
- x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
- x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
- zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
- bias = Bxy - Bxy.est.feasible,
- Bxy.est.feasible = Bxy.est.feasible,
- sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
-
- x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bxy.est.feasible),
- var.est = var(Bxy.est.feasible),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='x',
- method='Feasible'
- ),
- by=c('N','m')]
-
+summarize.estimator <- function(df, suffix='naive', coefname='x'){
- g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
- g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
- zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
- bias = Bgy - Bgy.est.feasible,
- Bgy.est.feasible = Bgy.est.feasible,
- sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
-
- g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bgy.est.feasible),
- var.est = var(Bgy.est.feasible),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='g',
- method='Feasible'
- ),
- by=c('N','m')]
+ part <- df[,c('N',
+ 'm',
+ 'Bxy',
+ paste0('B',coefname,'y.est.',suffix),
+ paste0('B',coefname,'y.ci.lower.',suffix),
+ paste0('B',coefname,'y.ci.upper.',suffix),
+ 'y_explained_variance',
+ 'Bzy',
+ 'accuracy_imbalance_difference'
+ ),
+ with=FALSE]
+ true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+ zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+ bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+ sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+ part <- part[,':='(true.in.ci = true.in.ci,
+ zero.in.ci = zero.in.ci,
+ bias=bias,
+ sign.correct =sign.correct)]
+
+ part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+ mean.bias = mean(bias),
+ mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
+ est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
+ N.sims = .N,
+ p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+ variable=coefname,
+ method=suffix
+ ),
+ by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance')
+ ]
+
+ return(part.plot)
+}
+
+build_plot_dataset <- function(df){
+
+ x.true <- summarize.estimator(df, 'true','x')
+ z.true <- summarize.estimator(df, 'true','z')
- x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
-
- x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
- zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
- bias = Bxy.est.true - Bxy.est.amelia.full,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
-
- x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.amelia.full),
- var.est = var(Bxy.est.amelia.full),
- N.sims = .N,
- variable='x',
- method='Multiple imputation'
- ),
- by=c('N','m')]
-
-
- g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
- g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
- zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
- bias = Bgy.est.amelia.full - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
-
- g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.amelia.full),
- var.est = var(Bgy.est.amelia.full),
- N.sims = .N,
- variable='g',
- method='Multiple imputation'
- ),
- by=c('N','m')]
-
- x.mle <- df[,.(N,m, Bxy.est.true, Bxy.est.mle, Bxy.ci.lower.mle, Bxy.ci.upper.mle)]
-
- x.mle <- x.mle[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.mle) & (Bxy.est.true <= Bxy.ci.upper.mle),
- zero.in.ci = (0 >= Bxy.ci.lower.mle) & (0 <= Bxy.ci.upper.mle),
- bias = Bxy.est.mle - Bxy.est.true,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mle))]
-
- x.mle.plot <- x.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.mle),
- var.est = var(Bxy.est.mle),
- N.sims = .N,
- variable='x',
- method='Maximum Likelihood'
- ),
- by=c('N','m')]
-
-
-
- g.mle <- df[,.(N,m, Bgy.est.true, Bgy.est.mle, Bgy.ci.lower.mle, Bgy.ci.upper.mle)]
-
- g.mle <- g.mle[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mle) & (Bgy.est.true <= Bgy.ci.upper.mle),
- zero.in.ci = (0 >= Bgy.ci.lower.mle) & (0 <= Bgy.ci.upper.mle),
- bias = Bgy.est.mle - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mle))]
-
- g.mle.plot <- g.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.mle),
- var.est = var(Bgy.est.mle),
- N.sims = .N,
- variable='g',
- method='Maximum Likelihood'
- ),
- by=c('N','m')]
-
-
-
-
- x.pseudo <- df[,.(N,m, Bxy.est.true, Bxy.est.pseudo, Bxy.ci.lower.pseudo, Bxy.ci.upper.pseudo)]
-
- x.pseudo <- x.pseudo[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.pseudo) & (Bxy.est.true <= Bxy.ci.upper.pseudo),
- zero.in.ci = (0 >= Bxy.ci.lower.pseudo) & (0 <= Bxy.ci.upper.pseudo),
- bias = Bxy.est.pseudo - Bxy.est.true,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.pseudo))]
-
- x.pseudo.plot <- x.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.pseudo),
- var.est = var(Bxy.est.pseudo),
- N.sims = .N,
- variable='x',
- method='Pseudo Likelihood'
- ),
- by=c('N','m')]
-
-
-
- g.pseudo <- df[,.(N,m, Bgy.est.true, Bgy.est.pseudo, Bgy.ci.lower.pseudo, Bgy.ci.upper.pseudo)]
-
- g.pseudo <- g.pseudo[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.pseudo) & (Bgy.est.true <= Bgy.ci.upper.pseudo),
- zero.in.ci = (0 >= Bgy.ci.lower.pseudo) & (0 <= Bgy.ci.upper.pseudo),
- bias = Bgy.est.pseudo - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.pseudo))]
-
- g.pseudo.plot <- g.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.pseudo),
- var.est = var(Bgy.est.pseudo),
- N.sims = .N,
- variable='g',
- method='Pseudo Likelihood'
- ),
- by=c('N','m')]
+ x.naive <- summarize.estimator(df, 'naive','x')
+ z.naive <- summarize.estimator(df, 'naive','z')
+
+ x.feasible <- summarize.estimator(df, 'feasible','x')
+ z.feasible <- summarize.estimator(df, 'feasible','z')
+
+ x.amelia.full <- summarize.estimator(df, 'amelia.full','x')
+ z.amelia.full <- summarize.estimator(df, 'amelia.full','z')
+ x.mle <- summarize.estimator(df, 'mle','x')
+ z.mle <- summarize.estimator(df, 'mle','z')
+ x.zhang <- summarize.estimator(df, 'zhang','x')
+ z.zhang <- summarize.estimator(df, 'zhang','z')
accuracy <- df[,mean(accuracy)]
- plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mle.plot, g.mle.plot, x.pseudo.plot, g.pseudo.plot, x.feasible.plot, g.feasible.plot),use.names=T)
+ plot.df <- rbindlist(list(x.true, z.true, x.naive,z.naive,x.amelia.full,z.amelia.full,x.mle, z.mle, x.zhang, z.zhang, x.feasible, z.feasible),use.names=T)
plot.df[,accuracy := accuracy]
df <- read_feather(args$infile)
plot.df <- build_plot_dataset(df)
+
remember(plot.df,args$name)
## df[gmm.ER_pval<0.05]
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & !is.na(p.true.in.ci) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=-0.05),linetype=2)
-
-
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
args <- parse_args(parser)
-build_plot_dataset <- function(df){
- x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
- x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
- zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
- bias = Bxy - Bxy.est.naive,
- Bxy.est.naive = Bxy.est.naive,
- sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
-
- x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bxy.est.naive),
- var.est = var(Bxy.est.naive),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='x',
- method='Naive'
- ),
- by=c('N','m')]
+summarize.estimator <- function(df, suffix='naive', coefname='x'){
+
+ part <- df[,c('N',
+ 'm',
+ 'Bxy',
+ paste0('B',coefname,'y.est.',suffix),
+ paste0('B',coefname,'y.ci.lower.',suffix),
+ paste0('B',coefname,'y.ci.upper.',suffix),
+ 'y_explained_variance',
+ 'Bzx',
+ 'Bzy',
+ 'accuracy_imbalance_difference'
+ ),
+ with=FALSE]
+
+ true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+ zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+ bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+ sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+ part <- part[,':='(true.in.ci = true.in.ci,
+ zero.in.ci = zero.in.ci,
+ bias=bias,
+ sign.correct =sign.correct)]
+
+ part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+ mean.bias = mean(bias),
+ mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
+ est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
+ N.sims = .N,
+ p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+ variable=coefname,
+ method=suffix
+ ),
+ by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
+ ]
+ return(part.plot)
+}
- g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
- g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
- zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
- bias = Bgy - Bgy.est.naive,
- Bgy.est.naive = Bgy.est.naive,
- sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
-
- g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bgy.est.naive),
- var.est = var(Bgy.est.naive),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='g',
- method='Naive'
- ),
- by=c('N','m')]
+build_plot_dataset <- function(df){
+ x.true <- summarize.estimator(df, 'true','x')
+
+ z.true <- summarize.estimator(df, 'true','z')
- x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
- x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
- zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
- bias = Bxy - Bxy.est.feasible,
- Bxy.est.feasible = Bxy.est.feasible,
- sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
-
- x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bxy.est.feasible),
- var.est = var(Bxy.est.feasible),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='x',
- method='Feasible'
- ),
- by=c('N','m')]
+ x.naive <- summarize.estimator(df, 'naive','x')
+ z.naive <- summarize.estimator(df,'naive','z')
- g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
- g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
- zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
- bias = Bgy - Bgy.est.feasible,
- Bgy.est.feasible = Bgy.est.feasible,
- sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
-
- g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(Bgy.est.feasible),
- var.est = var(Bgy.est.feasible),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable='g',
- method='Feasible'
- ),
- by=c('N','m')]
+ x.feasible <- summarize.estimator(df, 'feasible', 'x')
+
+ z.feasible <- summarize.estimator(df, 'feasible', 'z')
+
+ x.amelia.full <- summarize.estimator(df, 'amelia.full', 'x')
+
+ z.amelia.full <- summarize.estimator(df, 'amelia.full', 'z')
+ x.mecor <- summarize.estimator(df, 'mecor', 'x')
+ z.mecor <- summarize.estimator(df, 'mecor', 'z')
- x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
-
- x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
- zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
- bias = Bxy.est.true - Bxy.est.amelia.full,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
-
- x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.amelia.full),
- var.est = var(Bxy.est.amelia.full),
- N.sims = .N,
- variable='x',
- method='Multiple imputation'
- ),
- by=c('N','m')]
-
-
- g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
- g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
- zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
- bias = Bgy.est.amelia.full - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
-
- g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.amelia.full),
- var.est = var(Bgy.est.amelia.full),
- N.sims = .N,
- variable='g',
- method='Multiple imputation'
- ),
- by=c('N','m')]
-
- ## x.amelia.nok <- df[,.(N, m, Bxy.est.true, Bxy.est.amelia.nok, Bxy.ci.lower.amelia.nok, Bxy.ci.upper.amelia.nok)]
- ## x.amelia.nok <- x.amelia.nok[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.nok) & (Bxy.est.true <= Bxy.ci.upper.amelia.nok),
- ## zero.in.ci = (0 >= Bxy.ci.lower.amelia.nok) & (0 <= Bxy.ci.upper.amelia.nok),
- ## bias = Bxy.est.amelia.nok - Bxy.est.true,
- ## sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.nok))]
-
- ## x.amelia.nok.plot <- x.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- ## mean.bias = mean(bias),
- ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- ## mean.est = mean(Bxy.est.amelia.nok),
- ## var.est = var(Bxy.est.amelia.nok),
- ## N.sims = .N,
- ## variable='x',
- ## method='Multiple imputation (Classifier features unobserved)'
- ## ),
- ## by=c('N','m')]
-
- ## g.amelia.nok <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.nok, Bgy.ci.lower.amelia.nok, Bgy.ci.upper.amelia.nok)]
- ## g.amelia.nok <- g.amelia.nok[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.nok) & (Bgy.est.true <= Bgy.ci.upper.amelia.nok),
- ## zero.in.ci = (0 >= Bgy.ci.lower.amelia.nok) & (0 <= Bgy.ci.upper.amelia.nok),
- ## bias = Bgy.est.amelia.nok - Bgy.est.true,
- ## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.nok))]
-
- ## g.amelia.nok.plot <- g.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- ## mean.bias = mean(bias),
- ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- ## mean.est = mean(Bgy.est.amelia.nok),
- ## var.est = var(Bgy.est.amelia.nok),
- ## N.sims = .N,
- ## variable='g',
- ## method="Multiple imputation (Classifier features unobserved)"
- ## ),
- ## by=c('N','m')]
-
-
- x.mecor <- df[,.(N,m,Bxy.est.true, Bxy.est.mecor,Bxy.lower.mecor, Bxy.upper.mecor)]
-
- x.mecor <- x.mecor[,':='(true.in.ci = (Bxy.est.true >= Bxy.lower.mecor) & (Bxy.est.true <= Bxy.upper.mecor),
- zero.in.ci = (0 >= Bxy.lower.mecor) & (0 <= Bxy.upper.mecor),
- bias = Bxy.est.mecor - Bxy.est.true,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mecor))]
-
- x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.mecor),
- var.est = var(Bxy.est.mecor),
- N.sims = .N,
- variable='x',
- method='Regression Calibration'
- ),
- by=c('N','m')]
-
- g.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.lower.mecor, Bgy.upper.mecor)]
-
- g.mecor <- g.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.lower.mecor) & (Bgy.est.true <= Bgy.upper.mecor),
- zero.in.ci = (0 >= Bgy.lower.mecor) & (0 <= Bgy.upper.mecor),
- bias = Bgy.est.mecor - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
-
- g.mecor.plot <- g.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.mecor),
- var.est = var(Bgy.est.mecor),
- N.sims = .N,
- variable='g',
- method='Regression Calibration'
- ),
- by=c('N','m')]
-
- ## x.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.ci.lower.mecor, Bgy.ci.upper.mecor)]
-
- ## x.mecor <- x.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mecor) & (Bgy.est.true <= Bgy.ci.upper.mecor),
- ## zero.in.ci = (0 >= Bgy.ci.lower.mecor) & (0 <= Bgy.ci.upper.mecor),
- ## bias = Bgy.est.mecor - Bgy.est.true,
- ## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
-
- ## x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- ## mean.bias = mean(bias),
- ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- ## variable='g',
- ## method='Regression Calibration'
- ## ),
- ## by=c('N','m')]
-
-
- x.gmm <- df[,.(N,m,Bxy.est.true, Bxy.est.gmm,Bxy.ci.lower.gmm, Bxy.ci.upper.gmm)]
- x.gmm <- x.gmm[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.gmm) & (Bxy.est.true <= Bxy.ci.upper.gmm),
- zero.in.ci = (0 >= Bxy.ci.lower.gmm) & (0 <= Bxy.ci.upper.gmm),
- bias = Bxy.est.gmm - Bxy.est.true,
- sign.correct = sign(Bxy.est.true) == sign(Bxy.est.gmm))]
-
- x.gmm.plot <- x.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bxy.est.gmm),
-
- var.est = var(Bxy.est.gmm),
- N.sims = .N,
- variable='x',
- method='2SLS+gmm'
- ),
- by=c('N','m')]
-
- g.gmm <- df[,.(N,m,Bgy.est.true, Bgy.est.gmm,Bgy.ci.lower.gmm, Bgy.ci.upper.gmm)]
- g.gmm <- g.gmm[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.gmm) & (Bgy.est.true <= Bgy.ci.upper.gmm),
- zero.in.ci = (0 >= Bgy.ci.lower.gmm) & (0 <= Bgy.ci.upper.gmm),
- bias = Bgy.est.gmm - Bgy.est.true,
- sign.correct = sign(Bgy.est.true) == sign(Bgy.est.gmm))]
-
- g.gmm.plot <- g.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
- mean.bias = mean(bias),
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- mean.est = mean(Bgy.est.gmm),
- var.est = var(Bgy.est.gmm),
- N.sims = .N,
- variable='g',
- method='2SLS+gmm'
- ),
- by=c('N','m')]
+ x.mecor <- summarize.estimator(df, 'mecor', 'x')
- accuracy <- df[,mean(accuracy)]
+ z.mecor <- summarize.estimator(df, 'mecor', 'z')
- plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mecor.plot, g.mecor.plot, x.gmm.plot, g.gmm.plot, x.feasible.plot, g.feasible.plot),use.names=T)
+ x.mle <- summarize.estimator(df, 'mle', 'x')
- plot.df[,accuracy := accuracy]
+ z.mle <- summarize.estimator(df, 'mle', 'z')
+
+ x.zhang <- summarize.estimator(df, 'zhang', 'x')
- plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+ z.zhang <- summarize.estimator(df, 'zhang', 'z')
+
+ x.gmm <- summarize.estimator(df, 'gmm', 'x')
+ z.gmm <- summarize.estimator(df, 'gmm', 'z')
+
+ accuracy <- df[,mean(accuracy)]
+ plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.mecor, z.mecor, x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T)
+ plot.df[,accuracy := accuracy]
+ plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
return(plot.df)
}
-df <- read_feather(args$infile)
-plot.df <- build_plot_dataset(df)
+plot.df <- read_feather(args$infile)
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(plot.df)))
+ plot.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+ plot.df[,accuracy_imbalance_difference:=NA]
+
+unique(plot.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(plot.df)
+
remember(plot.df,args$name)
+#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
+
+## ## ## df[gmm.ER_pval<0.05]
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle),
+## var.est = var(Bxy.est.mle),
+## N.sims = .N,
+## variable='z',
+## method='Bespoke MLE'
+## ),
+## by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')]
+
+## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
-## df[gmm.ER_pval<0.05]
+## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle),
+## var.est = var(Bzy.est.mle),
+## N.sims = .N,
+## variable='z',
+## method='Bespoke MLE'
+## ),
+## by=c("N","m",'y_explained_variance','Bzx')]
+## plot.df <- z.mle.plot
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=0.2),linetype=2)
+## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
-## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
+## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
-## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
+## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
-## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
+## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
## Resources
#SBATCH --nodes=1
## Walltime (12 hours)
-#SBATCH --time=24:00:00
+#SBATCH --time=1:00:00
## Memory per node
#SBATCH --mem=8G
#SBATCH --cpus-per-task=1
#SBATCH --ntasks-per-node=1
#SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
#SBATCH --output=simulation_jobs/%A_%a.out
-#SBATCH --error=simulation_jobs/%A_%a.out
+#SBATCH --error=simulation_jobs/%A_%a.err
TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
TASK_CALL=$(sed -n ${TASK_NUM}p $2)
amelia.ncpus=1)
library(Amelia)
library(Zelig)
-library(stats4)
+library(bbmle)
+library(matrixStats) # for numerically stable logsumexps
+source("measerr_methods.R") ## for my more generic function.
## This uses the pseudolikelihood approach from Carroll page 349.
## assumes MAR
## assumes differential error, but that only depends on Y
## inefficient, because pseudolikelihood
-logistic.correction.pseudo <- function(df){
+
+## This uses the pseudo-likelihood approach from Carroll page 346.
+my.pseudo.mle <- function(df){
p1.est <- mean(df[w_pred==1]$y.obs==1,na.rm=T)
p0.est <- mean(df[w_pred==0]$y.obs==0,na.rm=T)
- nll <- function(B0, Bxy, Bgy){
- probs <- (1 - p0.est) + (p1.est + p0.est - 1)*plogis(B0 + Bxy * df$x + Bgy * df$g)
+ nll <- function(B0, Bxy, Bzy){
- part1 = sum(log(probs[df$w_pred == 1]))
- part2 = sum(log(1-probs[df$w_pred == 0]))
+ pw <- vector(mode='numeric',length=nrow(df))
+ dfw1 <- df[w_pred==1]
+ dfw0 <- df[w_pred==0]
+ pw[df$w_pred==1] <- plogis(B0 + Bxy * dfw1$x + Bzy * dfw1$z, log=T)
+ pw[df$w_pred==0] <- plogis(B0 + Bxy * dfw0$x + Bzy * dfw0$z, lower.tail=FALSE, log=T)
- return(-1*(part1 + part2))
+ probs <- colLogSumExps(rbind(log(1 - p0.est), log(p1.est + p0.est - 1) + pw))
+ return(-1*sum(probs))
}
- mlefit <- stats4::mle(minuslogl = nll, start = list(B0=0, Bxy=0.0, Bgy=0.0))
+ mlefit <- mle2(minuslogl = nll, start = list(B0=0.0, Bxy=0.0, Bzy=0.0), control=list(maxit=1e6),method='L-BFGS-B')
return(mlefit)
}
+
+## model from Zhang's arxiv paper, with predictions for y
+## Zhang got this model from Hausman 1998
+### I think this is actually eqivalent to the pseudo.mle method
+zhang.mle.iv <- function(df){
+ nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){
+ df.obs <- df[!is.na(x.obs)]
+ df.unobs <- df[is.na(x.obs)]
+
+ ## fpr = 1 - TNR
+ ### Problem: accounting for uncertainty in ppv / npv
+
+ ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T))
+ ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T))
+
+ ## fnr = 1 - TPR
+ ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
+ ll <- sum(ll.y.obs)
+ ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs)
+
+ # unobserved case; integrate out x
+ ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
+ ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
+
+ ## case x == 1
+ lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0))
+
+ ## case x == 0
+ lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
+
+ lls <- colLogSumExps(rbind(lls.x.1, lls.x.0))
+ ll <- ll + sum(lls)
+ return(-ll)
+ }
+ mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001),
+ upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B')
+ return(mlefit)
+}
+
+## this is equivalent to the pseudo-liklihood model from Carolla
+zhang.mle.dv <- function(df){
+
+ nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
+ df.obs <- df[!is.na(y.obs)]
+
+ ## fpr = 1 - TNR
+ ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
+ ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
+
+ # observed case
+ ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
+ ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
+ ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
+
+ ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
+
+ # unobserved case; integrate out y
+ ## case y = 1
+ ll.y.1 <- vector(mode='numeric', length=nrow(df))
+ pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
+ ## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
+ lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
+
+ ## case y = 0
+ ll.y.0 <- vector(mode='numeric', length=nrow(df))
+ pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
+
+ ## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
+ lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
+
+ lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
+ ll <- ll + sum(lls)
+ return(-ll)
+ }
+ mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
+ upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
+ return(mlefit)
+}
+
## This uses the likelihood approach from Carroll page 353.
## assumes that we have a good measurement error model
-logistic.correction.liklihood <- function(df){
+my.mle <- function(df){
## liklihood for observed responses
- nll <- function(B0, Bxy, Bgy, gamma0, gamma_y, gamma_g){
+ nll <- function(B0, Bxy, Bzy, gamma0, gamma_y, gamma_z, gamma_yz){
df.obs <- df[!is.na(y.obs)]
- p.y.obs <- plogis(B0 + Bxy * df.obs$x + Bgy*df.obs$g)
- p.y.obs[df.obs$y==0] <- 1-p.y.obs[df.obs$y==0]
- p.s.obs <- plogis(gamma0 + gamma_y * df.obs$y + gamma_g*df.obs$g)
- p.s.obs[df.obs$w_pred==0] <- 1 - p.s.obs[df.obs$w_pred==0]
+ yobs0 <- df.obs$y==0
+ yobs1 <- df.obs$y==1
+ p.y.obs <- vector(mode='numeric', length=nrow(df.obs))
+
+ p.y.obs[yobs1] <- plogis(B0 + Bxy * df.obs[yobs1]$x + Bzy*df.obs[yobs1]$z,log=T)
+ p.y.obs[yobs0] <- plogis(B0 + Bxy * df.obs[yobs0]$x + Bzy*df.obs[yobs0]$z,lower.tail=FALSE,log=T)
+
+ wobs0 <- df.obs$w_pred==0
+ wobs1 <- df.obs$w_pred==1
+ p.w.obs <- vector(mode='numeric', length=nrow(df.obs))
+
+ p.w.obs[wobs1] <- plogis(gamma0 + gamma_y * df.obs[wobs1]$y + gamma_z*df.obs[wobs1]$z + df.obs[wobs1]$z*df.obs[wobs1]$y* gamma_yz, log=T)
+ p.w.obs[wobs0] <- plogis(gamma0 + gamma_y * df.obs[wobs0]$y + gamma_z*df.obs[wobs0]$z + df.obs[wobs0]$z*df.obs[wobs0]$y* gamma_yz, lower.tail=FALSE, log=T)
- p.obs <- p.s.obs * p.y.obs
+ p.obs <- p.w.obs + p.y.obs
df.unobs <- df[is.na(y.obs)]
- p.unobs.1 <- plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g)*plogis(gamma0 + gamma_y + gamma_g*df.unobs$g)
- p.unobs.0 <- (1-plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g))*plogis(gamma0 + gamma_g*df.unobs$g)
- p.unobs <- p.unobs.1 + p.unobs.0
- p.unobs[df.unobs$w_pred==0] <- 1 - p.unobs[df.unobs$w_pred==0]
+ p.unobs.0 <- vector(mode='numeric',length=nrow(df.unobs))
+ p.unobs.1 <- vector(mode='numeric',length=nrow(df.unobs))
+
+ wunobs.0 <- df.unobs$w_pred == 0
+ wunobs.1 <- df.unobs$w_pred == 1
+
+ p.unobs.0[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.1]$z + df.unobs[wunobs.1]$z*gamma_yz, log=T)
+
+ p.unobs.0[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.0]$z + df.unobs[wunobs.0]$z*gamma_yz, lower.tail=FALSE, log=T)
+
+ p.unobs.1[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.1]$z, log=T)
+
+ p.unobs.1[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.0]$z, lower.tail=FALSE, log=T)
+
+ p.unobs <- colLogSumExps(rbind(p.unobs.1, p.unobs.0))
p <- c(p.obs, p.unobs)
- return(-1*(sum(log(p))))
+ return(-1*(sum(p)))
}
- mlefit <- stats4::mle(minuslogl = nll, start = list(B0=1, Bxy=0,Bgy=0, gamma0=5, gamma_y=0, gamma_g=0))
+ mlefit <- mle2(minuslogl = nll, start = list(B0=0, Bxy=0,Bzy=0, gamma0=0, gamma_y=0, gamma_z=0, gamma_yz=0), control=list(maxit=1e6),method='L-BFGS-B')
return(mlefit)
}
-
-logistic <- function(x) {1/(1+exp(-1*x))}
-
-run_simulation_depvar <- function(df, result){
+run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y){
accuracy <- df[,mean(w_pred==y)]
result <- append(result, list(accuracy=accuracy))
- (model.true <- glm(y ~ x + g, data=df,family=binomial(link='logit')))
+ (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))
true.ci.Bxy <- confint(model.true)['x',]
- true.ci.Bgy <- confint(model.true)['g',]
+ true.ci.Bzy <- confint(model.true)['z',]
result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
- Bgy.est.true=coef(model.true)['g'],
+ Bzy.est.true=coef(model.true)['z'],
Bxy.ci.upper.true = true.ci.Bxy[2],
Bxy.ci.lower.true = true.ci.Bxy[1],
- Bgy.ci.upper.true = true.ci.Bgy[2],
- Bgy.ci.lower.true = true.ci.Bgy[1]))
+ Bzy.ci.upper.true = true.ci.Bzy[2],
+ Bzy.ci.lower.true = true.ci.Bzy[1]))
- (model.feasible <- glm(y.obs~x+g,data=df,family=binomial(link='logit')))
+ (model.feasible <- glm(y.obs~x+z,data=df,family=binomial(link='logit')))
feasible.ci.Bxy <- confint(model.feasible)['x',]
result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x'],
Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
- feasible.ci.Bgy <- confint(model.feasible)['g',]
- result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
- Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
- Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
+ feasible.ci.Bzy <- confint(model.feasible)['z',]
+ result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
+ Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
+ Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))
- (model.naive <- glm(w_pred~x+g, data=df, family=binomial(link='logit')))
+ (model.naive <- glm(w_pred~x+z, data=df, family=binomial(link='logit')))
naive.ci.Bxy <- confint(model.naive)['x',]
- naive.ci.Bgy <- confint(model.naive)['g',]
+ naive.ci.Bzy <- confint(model.naive)['z',]
result <- append(result, list(Bxy.est.naive=coef(model.naive)['x'],
- Bgy.est.naive=coef(model.naive)['g'],
+ Bzy.est.naive=coef(model.naive)['z'],
Bxy.ci.upper.naive = naive.ci.Bxy[2],
Bxy.ci.lower.naive = naive.ci.Bxy[1],
- Bgy.ci.upper.naive = naive.ci.Bgy[2],
- Bgy.ci.lower.naive = naive.ci.Bgy[1]))
+ Bzy.ci.upper.naive = naive.ci.Bzy[2],
+ Bzy.ci.lower.naive = naive.ci.Bzy[1]))
- (model.naive.cont <- lm(w~x+g, data=df))
+ (model.naive.cont <- lm(w~x+z, data=df))
naivecont.ci.Bxy <- confint(model.naive.cont)['x',]
- naivecont.ci.Bgy <- confint(model.naive.cont)['g',]
+ naivecont.ci.Bzy <- confint(model.naive.cont)['z',]
## my implementatoin of liklihood based correction
- mod.caroll.lik <- logistic.correction.liklihood(df)
- coef <- coef(mod.caroll.lik)
- ci <- confint(mod.caroll.lik)
+ temp.df <- copy(df)
+ temp.df[,y:=y.obs]
+ mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+ fisher.info <- solve(mod.caroll.lik$hessian)
+ coef <- mod.caroll.lik$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
result <- append(result,
- list(Bxy.est.mle = coef['Bxy'],
- Bxy.ci.upper.mle = ci['Bxy','97.5 %'],
- Bxy.ci.lower.mle = ci['Bxy','2.5 %'],
- Bgy.est.mle = coef['Bgy'],
- Bgy.ci.upper.mle = ci['Bgy','97.5 %'],
- Bgy.ci.lower.mle = ci['Bgy','2.5 %']))
-
+ list(Bxy.est.mle = coef['x'],
+ Bxy.ci.upper.mle = ci.upper['x'],
+ Bxy.ci.lower.mle = ci.lower['x'],
+ Bzy.est.mle = coef['z'],
+ Bzy.ci.upper.mle = ci.upper['z'],
+ Bzy.ci.lower.mle = ci.lower['z']))
+
## my implementatoin of liklihood based correction
- mod.caroll.pseudo <- logistic.correction.pseudo(df)
- coef <- coef(mod.caroll.pseudo)
- ci <- confint(mod.caroll.pseudo)
+ mod.zhang <- zhang.mle.dv(df)
+ coef <- coef(mod.zhang)
+ ci <- confint(mod.zhang,method='quad')
result <- append(result,
- list(Bxy.est.pseudo = coef['Bxy'],
- Bxy.ci.upper.pseudo = ci['Bxy','97.5 %'],
- Bxy.ci.lower.pseudo = ci['Bxy','2.5 %'],
- Bgy.est.pseudo = coef['Bgy'],
- Bgy.ci.upper.pseudo = ci['Bgy','97.5 %'],
- Bgy.ci.lower.pseudo = ci['Bgy','2.5 %']))
+ list(Bxy.est.zhang = coef['Bxy'],
+ Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+ Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+ Bzy.est.zhang = coef['Bzy'],
+ Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+ Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
# amelia says use normal distribution for binary variables.
- amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w_pred'))
- mod.amelia.k <- zelig(y.obs~x+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
- (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+ tryCatch({
+ amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'))
+ mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+ (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+ est.x.mi <- coefse['x','Estimate']
+ est.x.se <- coefse['x','Std.Error']
+ result <- append(result,
+ list(Bxy.est.amelia.full = est.x.mi,
+ Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
+ Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
+ ))
+
+ est.z.mi <- coefse['z','Estimate']
+ est.z.se <- coefse['z','Std.Error']
+
+ result <- append(result,
+ list(Bzy.est.amelia.full = est.z.mi,
+ Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+ Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+ ))
+
+ },
+ error = function(e){
+ message("An error occurred:\n",e)
+ result$error <- paste0(result$error,'\n', e)
+ })
- est.x.mi <- coefse['x','Estimate']
- est.x.se <- coefse['x','Std.Error']
- result <- append(result,
- list(Bxy.est.amelia.full = est.x.mi,
- Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
- Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
- ))
-
- est.g.mi <- coefse['g','Estimate']
- est.g.se <- coefse['g','Std.Error']
-
- result <- append(result,
- list(Bgy.est.amelia.full = est.g.mi,
- Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
- Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
- ))
return(result)
}
-run_simulation <- function(df, result){
+
+## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle
+run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){
accuracy <- df[,mean(w_pred==x)]
result <- append(result, list(accuracy=accuracy))
- (model.true <- lm(y ~ x + g, data=df))
+ (model.true <- lm(y ~ x + z, data=df))
true.ci.Bxy <- confint(model.true)['x',]
- true.ci.Bgy <- confint(model.true)['g',]
+ true.ci.Bzy <- confint(model.true)['z',]
result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
- Bgy.est.true=coef(model.true)['g'],
+ Bzy.est.true=coef(model.true)['z'],
Bxy.ci.upper.true = true.ci.Bxy[2],
Bxy.ci.lower.true = true.ci.Bxy[1],
- Bgy.ci.upper.true = true.ci.Bgy[2],
- Bgy.ci.lower.true = true.ci.Bgy[1]))
+ Bzy.ci.upper.true = true.ci.Bzy[2],
+ Bzy.ci.lower.true = true.ci.Bzy[1]))
- (model.feasible <- lm(y~x.obs+g,data=df))
+ (model.feasible <- lm(y~x.obs+z,data=df))
feasible.ci.Bxy <- confint(model.feasible)['x.obs',]
result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x.obs'],
Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
- feasible.ci.Bgy <- confint(model.feasible)['g',]
- result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
- Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
- Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
+ feasible.ci.Bzy <- confint(model.feasible)['z',]
+ result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
+ Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
+ Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))
- (model.naive <- lm(y~w+g, data=df))
+ (model.naive <- lm(y~w_pred+z, data=df))
- naive.ci.Bxy <- confint(model.naive)['w',]
- naive.ci.Bgy <- confint(model.naive)['g',]
+ naive.ci.Bxy <- confint(model.naive)['w_pred',]
+ naive.ci.Bzy <- confint(model.naive)['z',]
- result <- append(result, list(Bxy.est.naive=coef(model.naive)['w'],
- Bgy.est.naive=coef(model.naive)['g'],
+ result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'],
+ Bzy.est.naive=coef(model.naive)['z'],
Bxy.ci.upper.naive = naive.ci.Bxy[2],
Bxy.ci.lower.naive = naive.ci.Bxy[1],
- Bgy.ci.upper.naive = naive.ci.Bgy[2],
- Bgy.ci.lower.naive = naive.ci.Bgy[1]))
+ Bzy.ci.upper.naive = naive.ci.Bzy[2],
+ Bzy.ci.lower.naive = naive.ci.Bzy[1]))
+ tryCatch({
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))
- mod.amelia.k <- zelig(y~x.obs+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+ mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
(coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
est.x.mi <- coefse['x.obs','Estimate']
Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
))
- est.g.mi <- coefse['g','Estimate']
- est.g.se <- coefse['g','Std.Error']
+ est.z.mi <- coefse['z','Estimate']
+ est.z.se <- coefse['z','Std.Error']
result <- append(result,
- list(Bgy.est.amelia.full = est.g.mi,
- Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
- Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
+ list(Bzy.est.amelia.full = est.z.mi,
+ Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+ Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
))
+ },
+ error = function(e){
+ message("An error occurred:\n",e)
+ result$error <-paste0(result$error,'\n', e)
+ }
+ )
+
+ tryCatch({
+ temp.df <- copy(df)
+ temp.df <- temp.df[,x:=x.obs]
+ mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+ fisher.info <- solve(mod.caroll.lik$hessian)
+ coef <- mod.caroll.lik$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+
+ result <- append(result,
+ list(Bxy.est.mle = coef['x'],
+ Bxy.ci.upper.mle = ci.upper['x'],
+ Bxy.ci.lower.mle = ci.lower['x'],
+ Bzy.est.mle = coef['z'],
+ Bzy.ci.upper.mle = ci.upper['z'],
+ Bzy.ci.lower.mle = ci.lower['z']))
+ },
+
+ error = function(e){
+ message("An error occurred:\n",e)
+ result$error <- paste0(result$error,'\n', e)
+ })
+
+ tryCatch({
+
+ mod.zhang.lik <- zhang.mle.iv(df)
+ coef <- coef(mod.zhang.lik)
+ ci <- confint(mod.zhang.lik,method='quad')
+ result <- append(result,
+ list(Bxy.est.zhang = coef['Bxy'],
+ Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+ Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+ Bzy.est.zhang = coef['Bzy'],
+ Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+ Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
+ },
+
+ error = function(e){
+ message("An error occurred:\n",e)
+ result$error <- paste0(result$error,'\n', e)
+ })
+
## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)
## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE)
df <- df[order(x.obs)]
y <- df[,y]
x <- df[,x.obs]
- g <- df[,g]
- w <- df[,w]
+ z <- df[,z]
+ w <- df[,w_pred]
# gmm gets pretty close
- (gmm.res <- predicted_covariates(y, x, g, w, v, train, p, max_iter=100, verbose=TRUE))
+ (gmm.res <- predicted_covariates(y, x, z, w, v, train, p, max_iter=100, verbose=TRUE))
result <- append(result,
list(Bxy.est.gmm = gmm.res$beta[1,1],
))
result <- append(result,
- list(Bgy.est.gmm = gmm.res$beta[2,1],
- Bgy.ci.upper.gmm = gmm.res$confint[2,2],
- Bgy.ci.lower.gmm = gmm.res$confint[2,1]))
+ list(Bzy.est.gmm = gmm.res$beta[2,1],
+ Bzy.ci.upper.gmm = gmm.res$confint[2,2],
+ Bzy.ci.lower.gmm = gmm.res$confint[2,1]))
- mod.calibrated.mle <- mecor(y ~ MeasError(w, reference = x.obs) + g, df, B=400, method='efficient')
+ tryCatch({
+ mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient')
(mod.calibrated.mle)
(mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
result <- append(result, list(
Bxy.est.mecor = mecor.ci['Estimate'],
- Bxy.upper.mecor = mecor.ci['UCI'],
- Bxy.lower.mecor = mecor.ci['LCI'])
+ Bxy.ci.upper.mecor = mecor.ci['UCI'],
+ Bxy.ci.lower.mecor = mecor.ci['LCI'])
)
- (mecor.ci <- summary(mod.calibrated.mle)$c$ci['g',])
+ (mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',])
result <- append(result, list(
- Bgy.est.mecor = mecor.ci['Estimate'],
- Bgy.upper.mecor = mecor.ci['UCI'],
- Bgy.lower.mecor = mecor.ci['LCI'])
+ Bzy.est.mecor = mecor.ci['Estimate'],
+ Bzy.ci.upper.mecor = mecor.ci['UCI'],
+ Bzy.ci.lower.mecor = mecor.ci['LCI'])
)
-
+ },
+ error = function(e){
+ message("An error occurred:\n",e)
+ result$error <- paste0(result$error, '\n', e)
+ }
+ )
## clean up memory
## rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))
projects / ml_measurement_error_public.git / commitdiff