From: Nathan TeBlunthuis Date: Fri, 15 Jul 2022 20:58:18 +0000 (-0700) Subject: update simulation code for examples 1-3 X-Git-Url: https://code.communitydata.science/ml_measurement_error_public.git/commitdiff_plain/46e2d1fe4876a9ed906b723f9e5f74fcc949e339?ds=inline;hp=cb3f850c24bdb1c84afaf4902b33061c4460a42d update simulation code for examples 1-3 --- diff --git a/simulations/01_two_covariates.R b/simulations/01_two_covariates.R index 7b8e12e..73e8939 100644 --- a/simulations/01_two_covariates.R +++ b/simulations/01_two_covariates.R @@ -1,8 +1,10 @@ -### EXAMPLE 2_b: demonstrates how measurement error can lead to a type sign error in a covariate -### This is the same as example 2, only instead of x->k we have k->x. -### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign. -### Even when you include the proxy variable in the regression. -### But with some ground truth and multiple imputation, you can fix it. +### EXAMPLE 2_b: demonstrates how measurement error can lead to a type +### sign error in a covariate This is the same as example 2, only +### instead of x->k we have k->x. Even when you have a good +### predictor, if it's biased against a covariate you can get the +### wrong sign. Even when you include the proxy variable in the +### regression. But with some ground truth and multiple imputation, +### you can fix it. library(argparser) library(mecor) @@ -12,9 +14,9 @@ library(filelock) library(arrow) library(Amelia) library(Zelig) + library(predictionError) -options(amelia.parallel="no", - amelia.ncpus=1) +options(amelia.parallel="no", amelia.ncpus=1) source("simulation_base.R") @@ -28,20 +30,18 @@ source("simulation_base.R") #### how much power do we get from the model in the first place? (sweeping N and m) #### -simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_variance=0.025, gx_explained_variance=0.15, prediction_accuracy=0.73, seed=1){ +simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, seed=1){ set.seed(seed) - g <- rbinom(N, 1, 0.5) - - x.var.epsilon <- var(Bgx *g) * ((1-gx_explained_variance)/gx_explained_variance) - x.epsilon <- rnorm(N,sd=sqrt(x.var.epsilon)) - xprime <- Bgx * g + x.epsilon - x <- as.integer(logistic(scale(xprime)) > 0.5) + z <- rbinom(N, 1, 0.5) + # x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance) + xprime <- Bzx * z #+ x.var.epsilon + x <- rbinom(N,1,plogis(xprime)) - y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bxy*x,Bgy*g)) * ((1-y_explained_variance)/y_explained_variance) + y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance) y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon)) - y <- Bgy * g + Bxy * x + y.epsilon + y <- Bzy * z + Bxy * x + y.epsilon - df <- data.table(x=x,xprime=xprime,y=y,g=g) + df <- data.table(x=x,y=y,z=z) if(m < N){ df <- df[sample(nrow(df), m), x.obs := x] @@ -49,42 +49,53 @@ simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_va df <- df[, x.obs := x] } - df <- df[,w_pred:=x] - df <- df[sample(1:N,(1-prediction_accuracy)*N),w_pred:=(w_pred-1)**2] - w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response') - df <- df[,':='(w=w, w_pred = w_pred)] + ## how can you make a model with a specific accuracy? + w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy + + ## how can you make a model with a specific accuracy, with a continuous latent variable. + # now it makes the same amount of mistake to each point, probably + # add mean0 noise to the odds. + + w.noisey.odds = rlogis(N,qlogis(w0)) + df[,w := plogis(w.noisey.odds)] + df[,w_pred:=as.integer(w > 0.5)] + (mean(df$x==df$w_pred)) return(df) } parser <- arg_parser("Simulate data and fit corrected models") -parser <- add_argument(parser, "--N", default=500, help="number of observations of w") -parser <- add_argument(parser, "--m", default=100, help="m the number of ground truth observations") -parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng') +parser <- add_argument(parser, "--N", default=1000, help="number of observations of w") +parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations") +parser <- add_argument(parser, "--seed", default=57, help='seed for the rng') parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather') -parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005) -parser <- add_argument(parser, "--gx_explained_variance", help='what proportion of the variance of x can be explained by g?', default=0.15) +parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05) +# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3) parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73) - +parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1) args <- parse_args(parser) B0 <- 0 -Bxy <- 0.2 -Bgy <- -0.2 -Bgx <- 0.4 +Bxy <- 0.3 +Bzy <- -0.3 +Bzx <- args$Bzx -df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, Bgx, seed=args$seed, y_explained_variance = args$y_explained_variance, gx_explained_variance = args$gx_explained_variance, prediction_accuracy=args$prediction_accuracy) +if (args$m < args$N){ -result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'Bgx'=Bgx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'gx_explained_variance' = args$gx_explained_variance, "prediction_accuracy"=args$prediction_accuracy) -outline <- run_simulation(df, result) + df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance, prediction_accuracy=args$prediction_accuracy) -outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) -if(file.exists(args$outfile)){ - logdata <- read_feather(args$outfile) - logdata <- rbind(logdata,as.data.table(outline)) -} else { - logdata <- as.data.table(outline) -} + result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="") -print(outline) -write_feather(logdata, args$outfile) -unlock(outfile_lock) + outline <- run_simulation(df, result) + + outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) + if(file.exists(args$outfile)){ + logdata <- read_feather(args$outfile) + logdata <- rbind(logdata,as.data.table(outline),fill=TRUE) + } else { + logdata <- as.data.table(outline) + } + + print(outline) + write_feather(logdata, args$outfile) + unlock(outfile_lock) +} diff --git a/simulations/02_indep_differential.R b/simulations/02_indep_differential.R index d4e0916..7e2e428 100644 --- a/simulations/02_indep_differential.R +++ b/simulations/02_indep_differential.R @@ -31,17 +31,17 @@ source("simulation_base.R") ## one way to do it is by adding correlation to x.obs and y that isn't in w. ## in other words, the model is missing an important feature of x.obs that's related to y. -simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){ +simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){ set.seed(seed) # make w and y dependent - g <- rbinom(N, 1, 0.5) - x <- rbinom(N, 1, 0.5) + z <- rbinom(N, 1, 0.5) + x <- rbinom(N, 1, Bzx * z + 0.5) - y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bgy*g,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance) + y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance) y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon)) - y <- Bgy * g + Bxy * x + y.epsilon - - df <- data.table(x=x,y=y,g=g) + y <- Bzy * z + Bxy * x + y.epsilon + + df <- data.table(x=x,y=y,z=z) if(m < N){ df <- df[sample(nrow(df), m), x.obs := x] @@ -49,61 +49,117 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025, df <- df[, x.obs := x] } - df <- df[,w_pred:=x] - - pg <- mean(g) - px <- mean(x) - accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2) + ## px <- mean(x) + ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2) - # this works because of conditional probability - accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg)) - accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0 + ## # this works because of conditional probability + ## accuracy_x0 <- prediction_accuracy / (px*(accuracy_imbalance_ratio) + (1-px)) + ## accuracy_x1 <- accuracy_imbalance_ratio * accuracy_x0 - dfg0 <- df[g==0] - ng0 <- nrow(dfg0) - dfg1 <- df[g==1] - ng1 <- nrow(dfg1) + ## x0 <- df[x==0]$x + ## x1 <- df[x==1]$x + ## nx1 <- nrow(df[x==1]) + ## nx0 <- nrow(df[x==0]) - dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2] - dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2] + ## yx0 <- df[x==0]$y + ## yx1 <- df[x==1]$y + + # tranform yz0.1 into a logistic distribution with mean accuracy_z0 + ## acc.x0 <- plogis(0.5*scale(yx0) + qlogis(accuracy_x0)) + ## acc.x1 <- plogis(1.5*scale(yx1) + qlogis(accuracy_x1)) - df <- rbind(dfg0,dfg1) + ## w0x0 <- (1-x0)**2 + (-1)**(1-x0) * acc.x0 + ## w0x1 <- (1-x1)**2 + (-1)**(1-x1) * acc.x1 + pz <- mean(z) + accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2) - w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response') - df <- df[,':='(w=w, w_pred = w_pred)] + # this works because of conditional probability + accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz)) + accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0 + + z0x0 <- df[(z==0) & (x==0)]$x + z0x1 <- df[(z==0) & (x==1)]$x + z1x0 <- df[(z==1) & (x==0)]$x + z1x1 <- df[(z==1) & (x==1)]$x + + yz0x0 <- df[(z==0) & (x==0)]$y + yz0x1 <- df[(z==0) & (x==1)]$y + yz1x0 <- df[(z==1) & (x==0)]$y + yz1x1 <- df[(z==1) & (x==1)]$y + + nz0x0 <- nrow(df[(z==0) & (x==0)]) + nz0x1 <- nrow(df[(z==0) & (x==1)]) + nz1x0 <- nrow(df[(z==1) & (x==0)]) + nz1x1 <- nrow(df[(z==1) & (x==1)]) + + yz1 <- df[z==1]$y + yz1 <- df[z==1]$y + + # tranform yz0.1 into a logistic distribution with mean accuracy_z0 + acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0)) + acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0)) + acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1)) + acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1)) + + w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0 + w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1 + w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0 + w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1 + + ##perrorz0 <- w0z0*(pyz0) + ##perrorz1 <- w0z1*(pyz1) + + w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0)) + w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1)) + w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0)) + w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1)) + + df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)] + df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)] + df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)] + df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)] + + df[,w_pred:=as.integer(w > 0.5)] + print(mean(df[z==0]$x == df[z==0]$w_pred)) + print(mean(df[z==1]$x == df[z==1]$w_pred)) + print(mean(df$w_pred == df$x)) return(df) } parser <- arg_parser("Simulate data and fit corrected models") -parser <- add_argument(parser, "--N", default=5000, help="number of observations of w") -parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations") -parser <- add_argument(parser, "--seed", default=432, help='seed for the rng') +parser <- add_argument(parser, "--N", default=1400, help="number of observations of w") +parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations") +parser <- add_argument(parser, "--seed", default=50, help='seed for the rng') parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather') parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01) parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73) parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3) +parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3) +parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3) + args <- parse_args(parser) B0 <- 0 -Bxy <- 0.2 -Bgy <- -0.2 - -df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference) +Bxy <- 0.3 +Bzy <- args$Bzy -result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference) +if(args$m < args$N){ + df <- simulate_data(args$N, args$m, B0, Bxy, args$Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference) -outline <- run_simulation_depvar(df=df, result) + result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, error='') + outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x+z+y+x:y, truth_formula=x~z) + + outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) + if(file.exists(args$outfile)){ + logdata <- read_feather(args$outfile) + logdata <- rbind(logdata,as.data.table(outline), fill=TRUE) + } else { + logdata <- as.data.table(outline) + } -outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) -if(file.exists(args$outfile)){ - logdata <- read_feather(args$outfile) - logdata <- rbind(logdata,as.data.table(outline)) -} else { - logdata <- as.data.table(outline) + print(outline) + write_feather(logdata, args$outfile) + unlock(outfile_lock) } - -print(outline) -write_feather(logdata, args$outfile) -unlock(outfile_lock) diff --git a/simulations/03_depvar_differential.R b/simulations/03_depvar_differential.R index d52afe7..872931f 100644 --- a/simulations/03_depvar_differential.R +++ b/simulations/03_depvar_differential.R @@ -31,18 +31,18 @@ source("simulation_base.R") ## one way to do it is by adding correlation to x.obs and y that isn't in w. ## in other words, the model is missing an important feature of x.obs that's related to y. -simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){ +simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){ set.seed(seed) # make w and y dependent - g <- rbinom(N, 1, 0.5) + z <- rbinom(N, 1, 0.5) x <- rbinom(N, 1, 0.5) - ystar <- Bgy * g + Bxy * x - y <- rbinom(N,1,logistic(ystar)) + ystar <- Bzy * z + Bxy * x + y <- rbinom(N,1,plogis(ystar)) - # glm(y ~ x + g, family="binomial") + # glm(y ~ x + z, family="binomial") - df <- data.table(x=x,y=y,ystar=ystar,g=g) + df <- data.table(x=x,y=y,ystar=ystar,z=z) if(m < N){ df <- df[sample(nrow(df), m), y.obs := y] @@ -52,36 +52,44 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, ac df <- df[,w_pred:=y] - pg <- mean(g) + pz <- mean(z) accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2) # this works because of conditional probability - accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg)) - accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0 + accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz)) + accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0 - dfg0 <- df[g==0] - ng0 <- nrow(dfg0) - dfg1 <- df[g==1] - ng1 <- nrow(dfg1) - dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2] - dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2] + yz0 <- df[z==0]$y + yz1 <- df[z==1]$y + nz1 <- nrow(df[z==1]) + nz0 <- nrow(df[z==0]) - df <- rbind(dfg0,dfg1) + acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0)) + acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1)) + + w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0 + w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1 + + w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0)) + w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1)) + df[z==0,w:=plogis(w0z0.noisy.odds)] + df[z==1,w:=plogis(w0z1.noisy.odds)] - wmod <- glm(y.obs ~ w_pred,data=df[!is.null(y.obs)],family=binomial(link='logit')) - w <- predict(wmod,df,type='response') + df[,w_pred:=as.integer(w > 0.5)] - df <- df[,':='(w=w)] + print(mean(df[y==0]$y == df[y==0]$w_pred)) + print(mean(df[y==1]$y == df[y==1]$w_pred)) + print(mean(df$w_pred == df$y)) return(df) } parser <- arg_parser("Simulate data and fit corrected models") -parser <- add_argument(parser, "--N", default=5000, help="number of observations of w") -parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations") -parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng') +parser <- add_argument(parser, "--N", default=1000, help="number of observations of w") +parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations") +parser <- add_argument(parser, "--seed", default=17, help='seed for the rng') parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather') parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005) parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73) @@ -90,24 +98,26 @@ parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much args <- parse_args(parser) B0 <- 0 -Bxy <- 0.2 -Bgy <- -0.2 +Bxy <- 0.7 +Bzy <- -0.7 -df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference) +if(args$m < args$N){ + df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference) -result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference) + result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference) -outline <- run_simulation_depvar(df=df, result) + outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x) + outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) -outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE) -if(file.exists(args$outfile)){ - logdata <- read_feather(args$outfile) - logdata <- rbind(logdata,as.data.table(outline)) -} else { - logdata <- as.data.table(outline) -} + if(file.exists(args$outfile)){ + logdata <- read_feather(args$outfile) + logdata <- rbind(logdata,as.data.table(outline),fill=TRUE) + } else { + logdata <- as.data.table(outline) + } -print(outline) -write_feather(logdata, args$outfile) -unlock(outfile_lock) + print(outline) + write_feather(logdata, args$outfile) + unlock(outfile_lock) +} diff --git a/simulations/Makefile b/simulations/Makefile index 2b18fea..dec7889 100644 --- a/simulations/Makefile +++ b/simulations/Makefile @@ -1,28 +1,42 @@ SHELL=bash -Ns=[500,1000,10000] -ms=[50, 100, 250, 500] +Ns=[1000,3600,14400] +ms=[75,150,300] seeds=[$(shell seq -s, 1 250)] +explained_variances=[0.1] + all:remembr.RDS -srun=srun -A comdata -p compute-bigmem --time=10:00:00 --mem 4G -c 1 +srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1 + + +joblists:example_1_jobs example_2_jobs example_3_jobs + +# test_true_z_jobs: test_true_z.R simulation_base.R +# grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb -example_1_jobs: 01_two_covariates.R - grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"]}' --outfile example_1_jobs +# test_true_z.feather: test_true_z_jobs +# rm -f test_true_z.feather +# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 test_true_z_jobs +# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs -example_1.feather: example_1_jobs + +example_1_jobs: 01_two_covariates.R simulation_base.R + grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs + +example_1.feather: example_1_jobs rm -f example_1.feather - sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_1_jobs - sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs + sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs +# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs -example_2_jobs: example_2.R - grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"]}' --outfile example_2_jobs +example_2_jobs: 02_indep_differential.R simulation_base.R + grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs -example_2.feather: example_2_jobs +example_2.feather: example_2_jobs rm -f example_2.feather - sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_jobs - sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs + sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs +# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs # example_2_B_jobs: example_2_B.R # grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs @@ -31,23 +45,24 @@ example_2.feather: example_2_jobs # rm -f example_2_B.feather # sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs -example_3_jobs: 03_depvar_differential.R - grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"]}' --outfile example_3_jobs +example_3_jobs: 03_depvar_differential.R simulation_base.R + grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs example_3.feather: example_3_jobs - rm -f example_3.feather - sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_3_jobs - sbatch --wait --verbose --array=3001-6000 run_simulation.sbatch 0 example_3_jobs + rm -f example_3.feather + sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs -remembr.RDS:example_1.feather example_2.feather example_3.feather + +remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R + rm -f remembr.RDS ${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1" ${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2" ${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3" clean: rm *.feather - rm remembr.RDS - + rm -f remembr.RDS + rm -f example_*_jobs # sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs # example_2_B_mecor_jobs: diff --git a/simulations/measerr_methods.R b/simulations/measerr_methods.R new file mode 100644 index 0000000..ab87d71 --- /dev/null +++ b/simulations/measerr_methods.R @@ -0,0 +1,227 @@ +library(formula.tools) +library(matrixStats) + +## df: dataframe to model +## outcome_formula: formula for y | x, z +## outcome_family: family for y | x, z +## proxy_formula: formula for w | x, z, y +## proxy_family: family for w | x, z, y +## truth_formula: formula for x | z +## truth_family: family for x | z + +### ideal formulas for example 1 +# test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z) + +### ideal formulas for example 2 +# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z) + + +## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y +measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){ + + nll <- function(params){ + df.obs <- model.frame(outcome_formula, df) + proxy.variable <- all.vars(proxy_formula)[1] + proxy.model.matrix <- model.matrix(proxy_formula, df) + response.var <- all.vars(outcome_formula)[1] + y.obs <- with(df.obs,eval(parse(text=response.var))) + outcome.model.matrix <- model.matrix(outcome_formula, df.obs) + + param.idx <- 1 + n.outcome.model.covars <- dim(outcome.model.matrix)[2] + outcome.params <- params[param.idx:n.outcome.model.covars] + param.idx <- param.idx + n.outcome.model.covars + + if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){ + ll.y.obs <- vector(mode='numeric', length=length(y.obs)) + ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE) + ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE) + } + + df.obs <- model.frame(proxy_formula,df) + n.proxy.model.covars <- dim(proxy.model.matrix)[2] + proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)] + + param.idx <- param.idx + n.proxy.model.covars + proxy.obs <- with(df.obs, eval(parse(text=proxy.variable))) + + if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){ + ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1]) + ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE) + ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE) + } + + ll.obs <- sum(ll.y.obs + ll.w.obs) + + df.unobs <- df[is.na(df[[response.var]])] + df.unobs.y1 <- copy(df.unobs) + df.unobs.y1[[response.var]] <- 1 + df.unobs.y0 <- copy(df.unobs) + df.unobs.y0[[response.var]] <- 1 + + ## integrate out y + outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1) + + if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){ + ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1]) + ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1]) + ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE) + ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE) + } + + proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1) + proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0) + proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable))) + + if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){ + ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1]) + ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1]) + ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE) + ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE) + + ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE) + ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE) + } + + ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1 + ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0 + ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0))) + ll <- ll.unobs + ll.obs + return(-ll) + } + + params <- colnames(model.matrix(outcome_formula,df)) + lower <- rep(-Inf, length(params)) + proxy.params <- colnames(model.matrix(proxy_formula, df)) + params <- c(params, paste0('proxy_',proxy.params)) + lower <- c(lower, rep(-Inf, length(proxy.params))) + start <- rep(0.1,length(params)) + names(start) <- params + + fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + return(fit) +} + +measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){ + + measrr_mle_nll <- function(params){ + df.obs <- model.frame(outcome_formula, df) + + proxy.variable <- all.vars(proxy_formula)[1] + proxy.model.matrix <- model.matrix(proxy_formula, df) + + response.var <- all.vars(outcome_formula)[1] + y.obs <- with(df.obs,eval(parse(text=response.var))) + + outcome.model.matrix <- model.matrix(outcome_formula, df) + + param.idx <- 1 + n.outcome.model.covars <- dim(outcome.model.matrix)[2] + outcome.params <- params[param.idx:n.outcome.model.covars] + param.idx <- param.idx + n.outcome.model.covars + + ## likelihood for the fully observed data + if(outcome_family$family == "gaussian"){ + sigma.y <- params[param.idx] + param.idx <- param.idx + 1 + ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE) + } + + df.obs <- model.frame(proxy_formula,df) + n.proxy.model.covars <- dim(proxy.model.matrix)[2] + proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)] + param.idx <- param.idx + n.proxy.model.covars + proxy.obs <- with(df.obs, eval(parse(text=proxy.variable))) + + if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){ + ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1]) + ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE) + ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE) + } + + df.obs <- model.frame(truth_formula, df) + truth.variable <- all.vars(truth_formula)[1] + truth.obs <- with(df.obs, eval(parse(text=truth.variable))) + truth.model.matrix <- model.matrix(truth_formula,df) + n.truth.model.covars <- dim(truth.model.matrix)[2] + + truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)] + + if( (truth_family$family=="binomial") & (truth_family$link=='logit')){ + ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1]) + ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE) + ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE) + } + + ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs) + + ## likelihood for the predicted data + ## integrate out the "truth" variable. + + if(truth_family$family=='binomial'){ + df.unobs <- df[is.na(eval(parse(text=truth.variable)))] + df.unobs.x1 <- copy(df.unobs) + df.unobs.x1[,'x'] <- 1 + df.unobs.x0 <- copy(df.unobs) + df.unobs.x0[,'x'] <- 0 + outcome.unobs <- with(df.unobs, eval(parse(text=response.var))) + + outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0) + outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1) + if(outcome_family$family=="gaussian"){ + ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE) + ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE) + } + + if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){ + + proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0) + proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1) + proxy.unobs <- df.unobs[[proxy.variable]] + ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1]) + ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1]) + + ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE) + ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE) + + ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE) + ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE) + } + + if(truth_family$link=='logit'){ + truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0) + ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE) + ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE) + } + } + + ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0 + ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1 + ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1))) + return(-(ll.unobs + ll.obs)) + } + + outcome.params <- colnames(model.matrix(outcome_formula,df)) + lower <- rep(-Inf, length(outcome.params)) + + if(outcome_family$family=='gaussian'){ + params <- c(outcome.params, 'sigma_y') + lower <- c(lower, 0.00001) + } else { + params <- outcome.params + } + + proxy.params <- colnames(model.matrix(proxy_formula, df)) + params <- c(params, paste0('proxy_',proxy.params)) + lower <- c(lower, rep(-Inf, length(proxy.params))) + + truth.params <- colnames(model.matrix(truth_formula, df)) + params <- c(params, paste0('truth_', truth.params)) + lower <- c(lower, rep(-Inf, length(truth.params))) + start <- rep(0.1,length(params)) + names(start) <- params + + fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + + return(fit) +} diff --git a/simulations/plot_dv_example.R b/simulations/plot_dv_example.R index 961bc87..f69ed6c 100644 --- a/simulations/plot_dv_example.R +++ b/simulations/plot_dv_example.R @@ -10,204 +10,73 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.") args <- parse_args(parser) -build_plot_dataset <- function(df){ - x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)] - x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)), - zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive), - bias = Bxy - Bxy.est.naive, - Bxy.est.naive = Bxy.est.naive, - sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))] - - x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bxy.est.naive), - var.est = var(Bxy.est.naive), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='x', - method='Naive' - ), - by=c('N','m')] - - g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)] - g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)), - zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive), - bias = Bgy - Bgy.est.naive, - Bgy.est.naive = Bgy.est.naive, - sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))] - - g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bgy.est.naive), - var.est = var(Bgy.est.naive), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='g', - method='Naive' - ), - by=c('N','m')] - - x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)] - x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)), - zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible), - bias = Bxy - Bxy.est.feasible, - Bxy.est.feasible = Bxy.est.feasible, - sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))] - - x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bxy.est.feasible), - var.est = var(Bxy.est.feasible), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='x', - method='Feasible' - ), - by=c('N','m')] - +summarize.estimator <- function(df, suffix='naive', coefname='x'){ - g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)] - g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)), - zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible), - bias = Bgy - Bgy.est.feasible, - Bgy.est.feasible = Bgy.est.feasible, - sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))] - - g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bgy.est.feasible), - var.est = var(Bgy.est.feasible), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='g', - method='Feasible' - ), - by=c('N','m')] + part <- df[,c('N', + 'm', + 'Bxy', + paste0('B',coefname,'y.est.',suffix), + paste0('B',coefname,'y.ci.lower.',suffix), + paste0('B',coefname,'y.ci.upper.',suffix), + 'y_explained_variance', + 'Bzy', + 'accuracy_imbalance_difference' + ), + with=FALSE] + true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) + zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) + bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] + sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) + + part <- part[,':='(true.in.ci = true.in.ci, + zero.in.ci = zero.in.ci, + bias=bias, + sign.correct =sign.correct)] + + part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), + mean.bias = mean(bias), + mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), + var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), + est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95), + est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05), + N.sims = .N, + p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), + variable=coefname, + method=suffix + ), + by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance') + ] + + return(part.plot) +} + +build_plot_dataset <- function(df){ + + x.true <- summarize.estimator(df, 'true','x') + z.true <- summarize.estimator(df, 'true','z') - x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)] - - x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full), - zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full), - bias = Bxy.est.true - Bxy.est.amelia.full, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))] - - x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.amelia.full), - var.est = var(Bxy.est.amelia.full), - N.sims = .N, - variable='x', - method='Multiple imputation' - ), - by=c('N','m')] - - - g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)] - g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full), - zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full), - bias = Bgy.est.amelia.full - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))] - - g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.amelia.full), - var.est = var(Bgy.est.amelia.full), - N.sims = .N, - variable='g', - method='Multiple imputation' - ), - by=c('N','m')] - - x.mle <- df[,.(N,m, Bxy.est.true, Bxy.est.mle, Bxy.ci.lower.mle, Bxy.ci.upper.mle)] - - x.mle <- x.mle[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.mle) & (Bxy.est.true <= Bxy.ci.upper.mle), - zero.in.ci = (0 >= Bxy.ci.lower.mle) & (0 <= Bxy.ci.upper.mle), - bias = Bxy.est.mle - Bxy.est.true, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mle))] - - x.mle.plot <- x.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.mle), - var.est = var(Bxy.est.mle), - N.sims = .N, - variable='x', - method='Maximum Likelihood' - ), - by=c('N','m')] - - - - g.mle <- df[,.(N,m, Bgy.est.true, Bgy.est.mle, Bgy.ci.lower.mle, Bgy.ci.upper.mle)] - - g.mle <- g.mle[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mle) & (Bgy.est.true <= Bgy.ci.upper.mle), - zero.in.ci = (0 >= Bgy.ci.lower.mle) & (0 <= Bgy.ci.upper.mle), - bias = Bgy.est.mle - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mle))] - - g.mle.plot <- g.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.mle), - var.est = var(Bgy.est.mle), - N.sims = .N, - variable='g', - method='Maximum Likelihood' - ), - by=c('N','m')] - - - - - x.pseudo <- df[,.(N,m, Bxy.est.true, Bxy.est.pseudo, Bxy.ci.lower.pseudo, Bxy.ci.upper.pseudo)] - - x.pseudo <- x.pseudo[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.pseudo) & (Bxy.est.true <= Bxy.ci.upper.pseudo), - zero.in.ci = (0 >= Bxy.ci.lower.pseudo) & (0 <= Bxy.ci.upper.pseudo), - bias = Bxy.est.pseudo - Bxy.est.true, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.pseudo))] - - x.pseudo.plot <- x.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.pseudo), - var.est = var(Bxy.est.pseudo), - N.sims = .N, - variable='x', - method='Pseudo Likelihood' - ), - by=c('N','m')] - - - - g.pseudo <- df[,.(N,m, Bgy.est.true, Bgy.est.pseudo, Bgy.ci.lower.pseudo, Bgy.ci.upper.pseudo)] - - g.pseudo <- g.pseudo[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.pseudo) & (Bgy.est.true <= Bgy.ci.upper.pseudo), - zero.in.ci = (0 >= Bgy.ci.lower.pseudo) & (0 <= Bgy.ci.upper.pseudo), - bias = Bgy.est.pseudo - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.pseudo))] - - g.pseudo.plot <- g.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.pseudo), - var.est = var(Bgy.est.pseudo), - N.sims = .N, - variable='g', - method='Pseudo Likelihood' - ), - by=c('N','m')] + x.naive <- summarize.estimator(df, 'naive','x') + z.naive <- summarize.estimator(df, 'naive','z') + + x.feasible <- summarize.estimator(df, 'feasible','x') + z.feasible <- summarize.estimator(df, 'feasible','z') + + x.amelia.full <- summarize.estimator(df, 'amelia.full','x') + z.amelia.full <- summarize.estimator(df, 'amelia.full','z') + x.mle <- summarize.estimator(df, 'mle','x') + z.mle <- summarize.estimator(df, 'mle','z') + x.zhang <- summarize.estimator(df, 'zhang','x') + z.zhang <- summarize.estimator(df, 'zhang','z') accuracy <- df[,mean(accuracy)] - plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mle.plot, g.mle.plot, x.pseudo.plot, g.pseudo.plot, x.feasible.plot, g.feasible.plot),use.names=T) + plot.df <- rbindlist(list(x.true, z.true, x.naive,z.naive,x.amelia.full,z.amelia.full,x.mle, z.mle, x.zhang, z.zhang, x.feasible, z.feasible),use.names=T) plot.df[,accuracy := accuracy] @@ -219,15 +88,22 @@ build_plot_dataset <- function(df){ df <- read_feather(args$infile) plot.df <- build_plot_dataset(df) + remember(plot.df,args$name) ## df[gmm.ER_pval<0.05] +## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T), +## N=factor(N), +## m=factor(m))] +## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & !is.na(p.true.in.ci) & (method!="Multiple imputation (Classifier features unobserved)")] +## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) +## p <- p + geom_hline(aes(yintercept=-0.05),linetype=2) - - +## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4)) +## print(p) ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10)) ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") diff --git a/simulations/plot_example.R b/simulations/plot_example.R index 1a4be9b..ebfd3a9 100644 --- a/simulations/plot_example.R +++ b/simulations/plot_example.R @@ -10,267 +10,172 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.") args <- parse_args(parser) -build_plot_dataset <- function(df){ - x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)] - x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)), - zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive), - bias = Bxy - Bxy.est.naive, - Bxy.est.naive = Bxy.est.naive, - sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))] - - x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bxy.est.naive), - var.est = var(Bxy.est.naive), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='x', - method='Naive' - ), - by=c('N','m')] +summarize.estimator <- function(df, suffix='naive', coefname='x'){ + + part <- df[,c('N', + 'm', + 'Bxy', + paste0('B',coefname,'y.est.',suffix), + paste0('B',coefname,'y.ci.lower.',suffix), + paste0('B',coefname,'y.ci.upper.',suffix), + 'y_explained_variance', + 'Bzx', + 'Bzy', + 'accuracy_imbalance_difference' + ), + with=FALSE] + + true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) + zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) + bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] + sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) + + part <- part[,':='(true.in.ci = true.in.ci, + zero.in.ci = zero.in.ci, + bias=bias, + sign.correct =sign.correct)] + + part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), + mean.bias = mean(bias), + mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), + var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), + est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T), + est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T), + N.sims = .N, + p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), + variable=coefname, + method=suffix + ), + by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference') + ] + return(part.plot) +} - g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)] - g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)), - zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive), - bias = Bgy - Bgy.est.naive, - Bgy.est.naive = Bgy.est.naive, - sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))] - - g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bgy.est.naive), - var.est = var(Bgy.est.naive), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='g', - method='Naive' - ), - by=c('N','m')] +build_plot_dataset <- function(df){ + x.true <- summarize.estimator(df, 'true','x') + + z.true <- summarize.estimator(df, 'true','z') - x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)] - x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)), - zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible), - bias = Bxy - Bxy.est.feasible, - Bxy.est.feasible = Bxy.est.feasible, - sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))] - - x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bxy.est.feasible), - var.est = var(Bxy.est.feasible), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='x', - method='Feasible' - ), - by=c('N','m')] + x.naive <- summarize.estimator(df, 'naive','x') + z.naive <- summarize.estimator(df,'naive','z') - g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)] - g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)), - zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible), - bias = Bgy - Bgy.est.feasible, - Bgy.est.feasible = Bgy.est.feasible, - sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))] - - g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(Bgy.est.feasible), - var.est = var(Bgy.est.feasible), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable='g', - method='Feasible' - ), - by=c('N','m')] + x.feasible <- summarize.estimator(df, 'feasible', 'x') + + z.feasible <- summarize.estimator(df, 'feasible', 'z') + + x.amelia.full <- summarize.estimator(df, 'amelia.full', 'x') + + z.amelia.full <- summarize.estimator(df, 'amelia.full', 'z') + x.mecor <- summarize.estimator(df, 'mecor', 'x') + z.mecor <- summarize.estimator(df, 'mecor', 'z') - x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)] - - x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full), - zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full), - bias = Bxy.est.true - Bxy.est.amelia.full, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))] - - x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.amelia.full), - var.est = var(Bxy.est.amelia.full), - N.sims = .N, - variable='x', - method='Multiple imputation' - ), - by=c('N','m')] - - - g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)] - g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full), - zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full), - bias = Bgy.est.amelia.full - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))] - - g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.amelia.full), - var.est = var(Bgy.est.amelia.full), - N.sims = .N, - variable='g', - method='Multiple imputation' - ), - by=c('N','m')] - - ## x.amelia.nok <- df[,.(N, m, Bxy.est.true, Bxy.est.amelia.nok, Bxy.ci.lower.amelia.nok, Bxy.ci.upper.amelia.nok)] - ## x.amelia.nok <- x.amelia.nok[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.nok) & (Bxy.est.true <= Bxy.ci.upper.amelia.nok), - ## zero.in.ci = (0 >= Bxy.ci.lower.amelia.nok) & (0 <= Bxy.ci.upper.amelia.nok), - ## bias = Bxy.est.amelia.nok - Bxy.est.true, - ## sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.nok))] - - ## x.amelia.nok.plot <- x.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - ## mean.bias = mean(bias), - ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - ## mean.est = mean(Bxy.est.amelia.nok), - ## var.est = var(Bxy.est.amelia.nok), - ## N.sims = .N, - ## variable='x', - ## method='Multiple imputation (Classifier features unobserved)' - ## ), - ## by=c('N','m')] - - ## g.amelia.nok <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.nok, Bgy.ci.lower.amelia.nok, Bgy.ci.upper.amelia.nok)] - ## g.amelia.nok <- g.amelia.nok[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.nok) & (Bgy.est.true <= Bgy.ci.upper.amelia.nok), - ## zero.in.ci = (0 >= Bgy.ci.lower.amelia.nok) & (0 <= Bgy.ci.upper.amelia.nok), - ## bias = Bgy.est.amelia.nok - Bgy.est.true, - ## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.nok))] - - ## g.amelia.nok.plot <- g.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - ## mean.bias = mean(bias), - ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - ## mean.est = mean(Bgy.est.amelia.nok), - ## var.est = var(Bgy.est.amelia.nok), - ## N.sims = .N, - ## variable='g', - ## method="Multiple imputation (Classifier features unobserved)" - ## ), - ## by=c('N','m')] - - - x.mecor <- df[,.(N,m,Bxy.est.true, Bxy.est.mecor,Bxy.lower.mecor, Bxy.upper.mecor)] - - x.mecor <- x.mecor[,':='(true.in.ci = (Bxy.est.true >= Bxy.lower.mecor) & (Bxy.est.true <= Bxy.upper.mecor), - zero.in.ci = (0 >= Bxy.lower.mecor) & (0 <= Bxy.upper.mecor), - bias = Bxy.est.mecor - Bxy.est.true, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mecor))] - - x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.mecor), - var.est = var(Bxy.est.mecor), - N.sims = .N, - variable='x', - method='Regression Calibration' - ), - by=c('N','m')] - - g.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.lower.mecor, Bgy.upper.mecor)] - - g.mecor <- g.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.lower.mecor) & (Bgy.est.true <= Bgy.upper.mecor), - zero.in.ci = (0 >= Bgy.lower.mecor) & (0 <= Bgy.upper.mecor), - bias = Bgy.est.mecor - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))] - - g.mecor.plot <- g.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.mecor), - var.est = var(Bgy.est.mecor), - N.sims = .N, - variable='g', - method='Regression Calibration' - ), - by=c('N','m')] - - ## x.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.ci.lower.mecor, Bgy.ci.upper.mecor)] - - ## x.mecor <- x.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mecor) & (Bgy.est.true <= Bgy.ci.upper.mecor), - ## zero.in.ci = (0 >= Bgy.ci.lower.mecor) & (0 <= Bgy.ci.upper.mecor), - ## bias = Bgy.est.mecor - Bgy.est.true, - ## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))] - - ## x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - ## mean.bias = mean(bias), - ## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - ## variable='g', - ## method='Regression Calibration' - ## ), - ## by=c('N','m')] - - - x.gmm <- df[,.(N,m,Bxy.est.true, Bxy.est.gmm,Bxy.ci.lower.gmm, Bxy.ci.upper.gmm)] - x.gmm <- x.gmm[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.gmm) & (Bxy.est.true <= Bxy.ci.upper.gmm), - zero.in.ci = (0 >= Bxy.ci.lower.gmm) & (0 <= Bxy.ci.upper.gmm), - bias = Bxy.est.gmm - Bxy.est.true, - sign.correct = sign(Bxy.est.true) == sign(Bxy.est.gmm))] - - x.gmm.plot <- x.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bxy.est.gmm), - - var.est = var(Bxy.est.gmm), - N.sims = .N, - variable='x', - method='2SLS+gmm' - ), - by=c('N','m')] - - g.gmm <- df[,.(N,m,Bgy.est.true, Bgy.est.gmm,Bgy.ci.lower.gmm, Bgy.ci.upper.gmm)] - g.gmm <- g.gmm[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.gmm) & (Bgy.est.true <= Bgy.ci.upper.gmm), - zero.in.ci = (0 >= Bgy.ci.lower.gmm) & (0 <= Bgy.ci.upper.gmm), - bias = Bgy.est.gmm - Bgy.est.true, - sign.correct = sign(Bgy.est.true) == sign(Bgy.est.gmm))] - - g.gmm.plot <- g.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)), - mean.bias = mean(bias), - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - mean.est = mean(Bgy.est.gmm), - var.est = var(Bgy.est.gmm), - N.sims = .N, - variable='g', - method='2SLS+gmm' - ), - by=c('N','m')] + x.mecor <- summarize.estimator(df, 'mecor', 'x') - accuracy <- df[,mean(accuracy)] + z.mecor <- summarize.estimator(df, 'mecor', 'z') - plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mecor.plot, g.mecor.plot, x.gmm.plot, g.gmm.plot, x.feasible.plot, g.feasible.plot),use.names=T) + x.mle <- summarize.estimator(df, 'mle', 'x') - plot.df[,accuracy := accuracy] + z.mle <- summarize.estimator(df, 'mle', 'z') + + x.zhang <- summarize.estimator(df, 'zhang', 'x') - plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)] + z.zhang <- summarize.estimator(df, 'zhang', 'z') + + x.gmm <- summarize.estimator(df, 'gmm', 'x') + z.gmm <- summarize.estimator(df, 'gmm', 'z') + + accuracy <- df[,mean(accuracy)] + plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.mecor, z.mecor, x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T) + plot.df[,accuracy := accuracy] + plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)] return(plot.df) } -df <- read_feather(args$infile) -plot.df <- build_plot_dataset(df) +plot.df <- read_feather(args$infile) + +# df <- df[apply(df,1,function(x) !any(is.na(x)))] + +if(!('Bzx' %in% names(plot.df))) + plot.df[,Bzx:=NA] + +if(!('accuracy_imbalance_difference' %in% names(plot.df))) + plot.df[,accuracy_imbalance_difference:=NA] + +unique(plot.df[,'accuracy_imbalance_difference']) + +#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700]) +plot.df <- build_plot_dataset(plot.df) + remember(plot.df,args$name) +#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy) + +## ## ## df[gmm.ER_pval<0.05] + +## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T), +## N=factor(N), +## m=factor(m))] + +## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")] +## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) +## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2) + +## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4)) +## print(p) + +## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T), +## N=factor(N), +## m=factor(m))] + +## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")] +## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) +## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2) + +## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4)) +## print(p) + + +## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)] +## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle), +## var.est = var(Bxy.est.mle), +## N.sims = .N, +## variable='z', +## method='Bespoke MLE' +## ), +## by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')] + +## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)] -## df[gmm.ER_pval<0.05] +## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle), +## var.est = var(Bzy.est.mle), +## N.sims = .N, +## variable='z', +## method='Bespoke MLE' +## ), +## by=c("N","m",'y_explained_variance','Bzx')] +## plot.df <- z.mle.plot +## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T), +## N=factor(N), +## m=factor(m))] +## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")] +## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) +## p <- p + geom_hline(aes(yintercept=0.2),linetype=2) +## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4)) +## print(p) -## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10)) +## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10)) -## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") +## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") -## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") +## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") diff --git a/simulations/run_simulation.sbatch b/simulations/run_simulation.sbatch index 835f39b..54f56be 100644 --- a/simulations/run_simulation.sbatch +++ b/simulations/run_simulation.sbatch @@ -6,14 +6,14 @@ ## Resources #SBATCH --nodes=1 ## Walltime (12 hours) -#SBATCH --time=24:00:00 +#SBATCH --time=1:00:00 ## Memory per node #SBATCH --mem=8G #SBATCH --cpus-per-task=1 #SBATCH --ntasks-per-node=1 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations #SBATCH --output=simulation_jobs/%A_%a.out -#SBATCH --error=simulation_jobs/%A_%a.out +#SBATCH --error=simulation_jobs/%A_%a.err TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1)) TASK_CALL=$(sed -n ${TASK_NUM}p $2) diff --git a/simulations/simulation_base.R b/simulations/simulation_base.R index a73ed79..0f03276 100644 --- a/simulations/simulation_base.R +++ b/simulations/simulation_base.R @@ -4,207 +4,324 @@ options(amelia.parallel="no", amelia.ncpus=1) library(Amelia) library(Zelig) -library(stats4) +library(bbmle) +library(matrixStats) # for numerically stable logsumexps +source("measerr_methods.R") ## for my more generic function. ## This uses the pseudolikelihood approach from Carroll page 349. ## assumes MAR ## assumes differential error, but that only depends on Y ## inefficient, because pseudolikelihood -logistic.correction.pseudo <- function(df){ + +## This uses the pseudo-likelihood approach from Carroll page 346. +my.pseudo.mle <- function(df){ p1.est <- mean(df[w_pred==1]$y.obs==1,na.rm=T) p0.est <- mean(df[w_pred==0]$y.obs==0,na.rm=T) - nll <- function(B0, Bxy, Bgy){ - probs <- (1 - p0.est) + (p1.est + p0.est - 1)*plogis(B0 + Bxy * df$x + Bgy * df$g) + nll <- function(B0, Bxy, Bzy){ - part1 = sum(log(probs[df$w_pred == 1])) - part2 = sum(log(1-probs[df$w_pred == 0])) + pw <- vector(mode='numeric',length=nrow(df)) + dfw1 <- df[w_pred==1] + dfw0 <- df[w_pred==0] + pw[df$w_pred==1] <- plogis(B0 + Bxy * dfw1$x + Bzy * dfw1$z, log=T) + pw[df$w_pred==0] <- plogis(B0 + Bxy * dfw0$x + Bzy * dfw0$z, lower.tail=FALSE, log=T) - return(-1*(part1 + part2)) + probs <- colLogSumExps(rbind(log(1 - p0.est), log(p1.est + p0.est - 1) + pw)) + return(-1*sum(probs)) } - mlefit <- stats4::mle(minuslogl = nll, start = list(B0=0, Bxy=0.0, Bgy=0.0)) + mlefit <- mle2(minuslogl = nll, start = list(B0=0.0, Bxy=0.0, Bzy=0.0), control=list(maxit=1e6),method='L-BFGS-B') return(mlefit) } + +## model from Zhang's arxiv paper, with predictions for y +## Zhang got this model from Hausman 1998 +### I think this is actually eqivalent to the pseudo.mle method +zhang.mle.iv <- function(df){ + nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){ + df.obs <- df[!is.na(x.obs)] + df.unobs <- df[is.na(x.obs)] + + ## fpr = 1 - TNR + ### Problem: accounting for uncertainty in ppv / npv + + ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T)) + ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T)) + + ## fnr = 1 - TPR + ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T)) + ll <- sum(ll.y.obs) + ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs) + + # unobserved case; integrate out x + ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T)) + ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T)) + + ## case x == 1 + lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0)) + + ## case x == 0 + lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0)) + + lls <- colLogSumExps(rbind(lls.x.1, lls.x.0)) + ll <- ll + sum(lls) + return(-ll) + } + mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001), + upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B') + return(mlefit) +} + +## this is equivalent to the pseudo-liklihood model from Carolla +zhang.mle.dv <- function(df){ + + nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){ + df.obs <- df[!is.na(y.obs)] + + ## fpr = 1 - TNR + ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE)) + ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE)) + + # observed case + ll.y.obs <- vector(mode='numeric', length=nrow(df.obs)) + ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T)) + ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE)) + + ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1) + + # unobserved case; integrate out y + ## case y = 1 + ll.y.1 <- vector(mode='numeric', length=nrow(df)) + pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T)) + ## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1) + lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1)) + + ## case y = 0 + ll.y.0 <- vector(mode='numeric', length=nrow(df)) + pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE)) + + ## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0) + lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0)) + + lls <- colLogSumExps(rbind(lls.y.1, lls.y.0)) + ll <- ll + sum(lls) + return(-ll) + } + mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001), + upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999)) + return(mlefit) +} + ## This uses the likelihood approach from Carroll page 353. ## assumes that we have a good measurement error model -logistic.correction.liklihood <- function(df){ +my.mle <- function(df){ ## liklihood for observed responses - nll <- function(B0, Bxy, Bgy, gamma0, gamma_y, gamma_g){ + nll <- function(B0, Bxy, Bzy, gamma0, gamma_y, gamma_z, gamma_yz){ df.obs <- df[!is.na(y.obs)] - p.y.obs <- plogis(B0 + Bxy * df.obs$x + Bgy*df.obs$g) - p.y.obs[df.obs$y==0] <- 1-p.y.obs[df.obs$y==0] - p.s.obs <- plogis(gamma0 + gamma_y * df.obs$y + gamma_g*df.obs$g) - p.s.obs[df.obs$w_pred==0] <- 1 - p.s.obs[df.obs$w_pred==0] + yobs0 <- df.obs$y==0 + yobs1 <- df.obs$y==1 + p.y.obs <- vector(mode='numeric', length=nrow(df.obs)) + + p.y.obs[yobs1] <- plogis(B0 + Bxy * df.obs[yobs1]$x + Bzy*df.obs[yobs1]$z,log=T) + p.y.obs[yobs0] <- plogis(B0 + Bxy * df.obs[yobs0]$x + Bzy*df.obs[yobs0]$z,lower.tail=FALSE,log=T) + + wobs0 <- df.obs$w_pred==0 + wobs1 <- df.obs$w_pred==1 + p.w.obs <- vector(mode='numeric', length=nrow(df.obs)) + + p.w.obs[wobs1] <- plogis(gamma0 + gamma_y * df.obs[wobs1]$y + gamma_z*df.obs[wobs1]$z + df.obs[wobs1]$z*df.obs[wobs1]$y* gamma_yz, log=T) + p.w.obs[wobs0] <- plogis(gamma0 + gamma_y * df.obs[wobs0]$y + gamma_z*df.obs[wobs0]$z + df.obs[wobs0]$z*df.obs[wobs0]$y* gamma_yz, lower.tail=FALSE, log=T) - p.obs <- p.s.obs * p.y.obs + p.obs <- p.w.obs + p.y.obs df.unobs <- df[is.na(y.obs)] - p.unobs.1 <- plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g)*plogis(gamma0 + gamma_y + gamma_g*df.unobs$g) - p.unobs.0 <- (1-plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g))*plogis(gamma0 + gamma_g*df.unobs$g) - p.unobs <- p.unobs.1 + p.unobs.0 - p.unobs[df.unobs$w_pred==0] <- 1 - p.unobs[df.unobs$w_pred==0] + p.unobs.0 <- vector(mode='numeric',length=nrow(df.unobs)) + p.unobs.1 <- vector(mode='numeric',length=nrow(df.unobs)) + + wunobs.0 <- df.unobs$w_pred == 0 + wunobs.1 <- df.unobs$w_pred == 1 + + p.unobs.0[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.1]$z + df.unobs[wunobs.1]$z*gamma_yz, log=T) + + p.unobs.0[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.0]$z + df.unobs[wunobs.0]$z*gamma_yz, lower.tail=FALSE, log=T) + + p.unobs.1[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.1]$z, log=T) + + p.unobs.1[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.0]$z, lower.tail=FALSE, log=T) + + p.unobs <- colLogSumExps(rbind(p.unobs.1, p.unobs.0)) p <- c(p.obs, p.unobs) - return(-1*(sum(log(p)))) + return(-1*(sum(p))) } - mlefit <- stats4::mle(minuslogl = nll, start = list(B0=1, Bxy=0,Bgy=0, gamma0=5, gamma_y=0, gamma_g=0)) + mlefit <- mle2(minuslogl = nll, start = list(B0=0, Bxy=0,Bzy=0, gamma0=0, gamma_y=0, gamma_z=0, gamma_yz=0), control=list(maxit=1e6),method='L-BFGS-B') return(mlefit) } - -logistic <- function(x) {1/(1+exp(-1*x))} - -run_simulation_depvar <- function(df, result){ +run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y){ accuracy <- df[,mean(w_pred==y)] result <- append(result, list(accuracy=accuracy)) - (model.true <- glm(y ~ x + g, data=df,family=binomial(link='logit'))) + (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit'))) true.ci.Bxy <- confint(model.true)['x',] - true.ci.Bgy <- confint(model.true)['g',] + true.ci.Bzy <- confint(model.true)['z',] result <- append(result, list(Bxy.est.true=coef(model.true)['x'], - Bgy.est.true=coef(model.true)['g'], + Bzy.est.true=coef(model.true)['z'], Bxy.ci.upper.true = true.ci.Bxy[2], Bxy.ci.lower.true = true.ci.Bxy[1], - Bgy.ci.upper.true = true.ci.Bgy[2], - Bgy.ci.lower.true = true.ci.Bgy[1])) + Bzy.ci.upper.true = true.ci.Bzy[2], + Bzy.ci.lower.true = true.ci.Bzy[1])) - (model.feasible <- glm(y.obs~x+g,data=df,family=binomial(link='logit'))) + (model.feasible <- glm(y.obs~x+z,data=df,family=binomial(link='logit'))) feasible.ci.Bxy <- confint(model.feasible)['x',] result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x'], Bxy.ci.upper.feasible = feasible.ci.Bxy[2], Bxy.ci.lower.feasible = feasible.ci.Bxy[1])) - feasible.ci.Bgy <- confint(model.feasible)['g',] - result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'], - Bgy.ci.upper.feasible = feasible.ci.Bgy[2], - Bgy.ci.lower.feasible = feasible.ci.Bgy[1])) + feasible.ci.Bzy <- confint(model.feasible)['z',] + result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'], + Bzy.ci.upper.feasible = feasible.ci.Bzy[2], + Bzy.ci.lower.feasible = feasible.ci.Bzy[1])) - (model.naive <- glm(w_pred~x+g, data=df, family=binomial(link='logit'))) + (model.naive <- glm(w_pred~x+z, data=df, family=binomial(link='logit'))) naive.ci.Bxy <- confint(model.naive)['x',] - naive.ci.Bgy <- confint(model.naive)['g',] + naive.ci.Bzy <- confint(model.naive)['z',] result <- append(result, list(Bxy.est.naive=coef(model.naive)['x'], - Bgy.est.naive=coef(model.naive)['g'], + Bzy.est.naive=coef(model.naive)['z'], Bxy.ci.upper.naive = naive.ci.Bxy[2], Bxy.ci.lower.naive = naive.ci.Bxy[1], - Bgy.ci.upper.naive = naive.ci.Bgy[2], - Bgy.ci.lower.naive = naive.ci.Bgy[1])) + Bzy.ci.upper.naive = naive.ci.Bzy[2], + Bzy.ci.lower.naive = naive.ci.Bzy[1])) - (model.naive.cont <- lm(w~x+g, data=df)) + (model.naive.cont <- lm(w~x+z, data=df)) naivecont.ci.Bxy <- confint(model.naive.cont)['x',] - naivecont.ci.Bgy <- confint(model.naive.cont)['g',] + naivecont.ci.Bzy <- confint(model.naive.cont)['z',] ## my implementatoin of liklihood based correction - mod.caroll.lik <- logistic.correction.liklihood(df) - coef <- coef(mod.caroll.lik) - ci <- confint(mod.caroll.lik) + temp.df <- copy(df) + temp.df[,y:=y.obs] + mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula) + fisher.info <- solve(mod.caroll.lik$hessian) + coef <- mod.caroll.lik$par + ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96 + ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96 result <- append(result, - list(Bxy.est.mle = coef['Bxy'], - Bxy.ci.upper.mle = ci['Bxy','97.5 %'], - Bxy.ci.lower.mle = ci['Bxy','2.5 %'], - Bgy.est.mle = coef['Bgy'], - Bgy.ci.upper.mle = ci['Bgy','97.5 %'], - Bgy.ci.lower.mle = ci['Bgy','2.5 %'])) - + list(Bxy.est.mle = coef['x'], + Bxy.ci.upper.mle = ci.upper['x'], + Bxy.ci.lower.mle = ci.lower['x'], + Bzy.est.mle = coef['z'], + Bzy.ci.upper.mle = ci.upper['z'], + Bzy.ci.lower.mle = ci.lower['z'])) + ## my implementatoin of liklihood based correction - mod.caroll.pseudo <- logistic.correction.pseudo(df) - coef <- coef(mod.caroll.pseudo) - ci <- confint(mod.caroll.pseudo) + mod.zhang <- zhang.mle.dv(df) + coef <- coef(mod.zhang) + ci <- confint(mod.zhang,method='quad') result <- append(result, - list(Bxy.est.pseudo = coef['Bxy'], - Bxy.ci.upper.pseudo = ci['Bxy','97.5 %'], - Bxy.ci.lower.pseudo = ci['Bxy','2.5 %'], - Bgy.est.pseudo = coef['Bgy'], - Bgy.ci.upper.pseudo = ci['Bgy','97.5 %'], - Bgy.ci.lower.pseudo = ci['Bgy','2.5 %'])) + list(Bxy.est.zhang = coef['Bxy'], + Bxy.ci.upper.zhang = ci['Bxy','97.5 %'], + Bxy.ci.lower.zhang = ci['Bxy','2.5 %'], + Bzy.est.zhang = coef['Bzy'], + Bzy.ci.upper.zhang = ci['Bzy','97.5 %'], + Bzy.ci.lower.zhang = ci['Bzy','2.5 %'])) # amelia says use normal distribution for binary variables. - amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w_pred')) - mod.amelia.k <- zelig(y.obs~x+g, model='ls', data=amelia.out.k$imputations, cite=FALSE) - (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE)) + tryCatch({ + amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w')) + mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE) + (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE)) + est.x.mi <- coefse['x','Estimate'] + est.x.se <- coefse['x','Std.Error'] + result <- append(result, + list(Bxy.est.amelia.full = est.x.mi, + Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se, + Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se + )) + + est.z.mi <- coefse['z','Estimate'] + est.z.se <- coefse['z','Std.Error'] + + result <- append(result, + list(Bzy.est.amelia.full = est.z.mi, + Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se, + Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se + )) + + }, + error = function(e){ + message("An error occurred:\n",e) + result$error <- paste0(result$error,'\n', e) + }) - est.x.mi <- coefse['x','Estimate'] - est.x.se <- coefse['x','Std.Error'] - result <- append(result, - list(Bxy.est.amelia.full = est.x.mi, - Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se, - Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se - )) - - est.g.mi <- coefse['g','Estimate'] - est.g.se <- coefse['g','Std.Error'] - - result <- append(result, - list(Bgy.est.amelia.full = est.g.mi, - Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se, - Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se - )) return(result) } -run_simulation <- function(df, result){ + +## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle +run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){ accuracy <- df[,mean(w_pred==x)] result <- append(result, list(accuracy=accuracy)) - (model.true <- lm(y ~ x + g, data=df)) + (model.true <- lm(y ~ x + z, data=df)) true.ci.Bxy <- confint(model.true)['x',] - true.ci.Bgy <- confint(model.true)['g',] + true.ci.Bzy <- confint(model.true)['z',] result <- append(result, list(Bxy.est.true=coef(model.true)['x'], - Bgy.est.true=coef(model.true)['g'], + Bzy.est.true=coef(model.true)['z'], Bxy.ci.upper.true = true.ci.Bxy[2], Bxy.ci.lower.true = true.ci.Bxy[1], - Bgy.ci.upper.true = true.ci.Bgy[2], - Bgy.ci.lower.true = true.ci.Bgy[1])) + Bzy.ci.upper.true = true.ci.Bzy[2], + Bzy.ci.lower.true = true.ci.Bzy[1])) - (model.feasible <- lm(y~x.obs+g,data=df)) + (model.feasible <- lm(y~x.obs+z,data=df)) feasible.ci.Bxy <- confint(model.feasible)['x.obs',] result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x.obs'], Bxy.ci.upper.feasible = feasible.ci.Bxy[2], Bxy.ci.lower.feasible = feasible.ci.Bxy[1])) - feasible.ci.Bgy <- confint(model.feasible)['g',] - result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'], - Bgy.ci.upper.feasible = feasible.ci.Bgy[2], - Bgy.ci.lower.feasible = feasible.ci.Bgy[1])) + feasible.ci.Bzy <- confint(model.feasible)['z',] + result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'], + Bzy.ci.upper.feasible = feasible.ci.Bzy[2], + Bzy.ci.lower.feasible = feasible.ci.Bzy[1])) - (model.naive <- lm(y~w+g, data=df)) + (model.naive <- lm(y~w_pred+z, data=df)) - naive.ci.Bxy <- confint(model.naive)['w',] - naive.ci.Bgy <- confint(model.naive)['g',] + naive.ci.Bxy <- confint(model.naive)['w_pred',] + naive.ci.Bzy <- confint(model.naive)['z',] - result <- append(result, list(Bxy.est.naive=coef(model.naive)['w'], - Bgy.est.naive=coef(model.naive)['g'], + result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'], + Bzy.est.naive=coef(model.naive)['z'], Bxy.ci.upper.naive = naive.ci.Bxy[2], Bxy.ci.lower.naive = naive.ci.Bxy[1], - Bgy.ci.upper.naive = naive.ci.Bgy[2], - Bgy.ci.lower.naive = naive.ci.Bgy[1])) + Bzy.ci.upper.naive = naive.ci.Bzy[2], + Bzy.ci.lower.naive = naive.ci.Bzy[1])) + tryCatch({ amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred')) - mod.amelia.k <- zelig(y~x.obs+g, model='ls', data=amelia.out.k$imputations, cite=FALSE) + mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE) (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE)) est.x.mi <- coefse['x.obs','Estimate'] @@ -215,15 +332,65 @@ run_simulation <- function(df, result){ Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se )) - est.g.mi <- coefse['g','Estimate'] - est.g.se <- coefse['g','Std.Error'] + est.z.mi <- coefse['z','Estimate'] + est.z.se <- coefse['z','Std.Error'] result <- append(result, - list(Bgy.est.amelia.full = est.g.mi, - Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se, - Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se + list(Bzy.est.amelia.full = est.z.mi, + Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se, + Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se )) + }, + error = function(e){ + message("An error occurred:\n",e) + result$error <-paste0(result$error,'\n', e) + } + ) + + tryCatch({ + temp.df <- copy(df) + temp.df <- temp.df[,x:=x.obs] + mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula) + fisher.info <- solve(mod.caroll.lik$hessian) + coef <- mod.caroll.lik$par + ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96 + ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96 + + + result <- append(result, + list(Bxy.est.mle = coef['x'], + Bxy.ci.upper.mle = ci.upper['x'], + Bxy.ci.lower.mle = ci.lower['x'], + Bzy.est.mle = coef['z'], + Bzy.ci.upper.mle = ci.upper['z'], + Bzy.ci.lower.mle = ci.lower['z'])) + }, + + error = function(e){ + message("An error occurred:\n",e) + result$error <- paste0(result$error,'\n', e) + }) + + tryCatch({ + + mod.zhang.lik <- zhang.mle.iv(df) + coef <- coef(mod.zhang.lik) + ci <- confint(mod.zhang.lik,method='quad') + result <- append(result, + list(Bxy.est.zhang = coef['Bxy'], + Bxy.ci.upper.zhang = ci['Bxy','97.5 %'], + Bxy.ci.lower.zhang = ci['Bxy','2.5 %'], + Bzy.est.zhang = coef['Bzy'], + Bzy.ci.upper.zhang = ci['Bzy','97.5 %'], + Bzy.ci.lower.zhang = ci['Bzy','2.5 %'])) + }, + + error = function(e){ + message("An error occurred:\n",e) + result$error <- paste0(result$error,'\n', e) + }) + ## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model. ## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms) ## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE) @@ -255,10 +422,10 @@ run_simulation <- function(df, result){ df <- df[order(x.obs)] y <- df[,y] x <- df[,x.obs] - g <- df[,g] - w <- df[,w] + z <- df[,z] + w <- df[,w_pred] # gmm gets pretty close - (gmm.res <- predicted_covariates(y, x, g, w, v, train, p, max_iter=100, verbose=TRUE)) + (gmm.res <- predicted_covariates(y, x, z, w, v, train, p, max_iter=100, verbose=TRUE)) result <- append(result, list(Bxy.est.gmm = gmm.res$beta[1,1], @@ -268,28 +435,34 @@ run_simulation <- function(df, result){ )) result <- append(result, - list(Bgy.est.gmm = gmm.res$beta[2,1], - Bgy.ci.upper.gmm = gmm.res$confint[2,2], - Bgy.ci.lower.gmm = gmm.res$confint[2,1])) + list(Bzy.est.gmm = gmm.res$beta[2,1], + Bzy.ci.upper.gmm = gmm.res$confint[2,2], + Bzy.ci.lower.gmm = gmm.res$confint[2,1])) - mod.calibrated.mle <- mecor(y ~ MeasError(w, reference = x.obs) + g, df, B=400, method='efficient') + tryCatch({ + mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient') (mod.calibrated.mle) (mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',]) result <- append(result, list( Bxy.est.mecor = mecor.ci['Estimate'], - Bxy.upper.mecor = mecor.ci['UCI'], - Bxy.lower.mecor = mecor.ci['LCI']) + Bxy.ci.upper.mecor = mecor.ci['UCI'], + Bxy.ci.lower.mecor = mecor.ci['LCI']) ) - (mecor.ci <- summary(mod.calibrated.mle)$c$ci['g',]) + (mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',]) result <- append(result, list( - Bgy.est.mecor = mecor.ci['Estimate'], - Bgy.upper.mecor = mecor.ci['UCI'], - Bgy.lower.mecor = mecor.ci['LCI']) + Bzy.est.mecor = mecor.ci['Estimate'], + Bzy.ci.upper.mecor = mecor.ci['UCI'], + Bzy.ci.lower.mecor = mecor.ci['LCI']) ) - + }, + error = function(e){ + message("An error occurred:\n",e) + result$error <- paste0(result$error, '\n', e) + } + ) ## clean up memory ## rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))