update simulation code for examples 1-3

diff --git a/simulations/01_two_covariates.R b/simulations/01_two_covariates.R
index 7b8e12ee90f53f855782561209f6b83505967fc7..73e8939abda420198b2a4c5a99f5d8d6fdff19a9 100644 (file)
--- a/simulations/01_two_covariates.R
+++ b/simulations/01_two_covariates.R
@@ -1,8 +1,10 @@
-### EXAMPLE 2_b: demonstrates how measurement error can lead to a type sign error in a covariate
-### This is the same as example 2, only instead of x->k we have k->x.
-### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
-### Even when you include the proxy variable in the regression.
-### But with some ground truth and multiple imputation, you can fix it.
+### EXAMPLE 2_b: demonstrates how measurement error can lead to a type
+### sign error in a covariate This is the same as example 2, only
+### instead of x->k we have k->x.  Even when you have a good
+### predictor, if it's biased against a covariate you can get the
+### wrong sign.  Even when you include the proxy variable in the
+### regression.  But with some ground truth and multiple imputation,
+### you can fix it.

 
 library(argparser)
 library(mecor)
 
 library(argparser)
 library(mecor)


@@ -12,9 +14,9 @@ library(filelock)
 library(arrow)
 library(Amelia)
 library(Zelig)
 library(arrow)
 library(Amelia)
 library(Zelig)

+

 library(predictionError)
 library(predictionError)

-options(amelia.parallel="no",
-        amelia.ncpus=1)
+options(amelia.parallel="no", amelia.ncpus=1)

 
 source("simulation_base.R")
 
 
 source("simulation_base.R")
 


@@ -28,20 +30,18 @@ source("simulation_base.R")
 #### how much power do we get from the model in the first place? (sweeping N and m)
 #### 
 
 #### how much power do we get from the model in the first place? (sweeping N and m)
 #### 
 

-simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_variance=0.025, gx_explained_variance=0.15, prediction_accuracy=0.73, seed=1){
+simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, seed=1){

     set.seed(seed)
     set.seed(seed)

-    g <- rbinom(N, 1, 0.5)
-
-    x.var.epsilon <- var(Bgx *g) * ((1-gx_explained_variance)/gx_explained_variance)
-    x.epsilon <- rnorm(N,sd=sqrt(x.var.epsilon))
-    xprime <- Bgx * g + x.epsilon
-    x <- as.integer(logistic(scale(xprime)) > 0.5)
+    z <- rbinom(N, 1, 0.5)
+                                        #    x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
+    xprime <- Bzx * z #+ x.var.epsilon
+    x <- rbinom(N,1,plogis(xprime))

 
 

-    y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bxy*x,Bgy*g)) * ((1-y_explained_variance)/y_explained_variance)
+    y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)

     y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
     y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))

-    y <- Bgy * g + Bxy * x + y.epsilon
+    y <- Bzy * z + Bxy * x + y.epsilon

 
 

-    df <- data.table(x=x,xprime=xprime,y=y,g=g)
+    df <- data.table(x=x,y=y,z=z)

 
     if(m < N){
         df <- df[sample(nrow(df), m), x.obs := x]
 
     if(m < N){
         df <- df[sample(nrow(df), m), x.obs := x]


@@ -49,42 +49,53 @@ simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_va
         df <- df[, x.obs := x]
     }
 
         df <- df[, x.obs := x]
     }
 

-    df <- df[,w_pred:=x]
-    df <- df[sample(1:N,(1-prediction_accuracy)*N),w_pred:=(w_pred-1)**2]
-    w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
-    df <- df[,':='(w=w, w_pred = w_pred)]
+    ## how can you make a model with a specific accuracy?
+    w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy
+
+    ## how can you make a model with a specific accuracy, with a continuous latent variable.
+    # now it makes the same amount of mistake to each point, probably
+    # add mean0 noise to the odds.
+    
+    w.noisey.odds = rlogis(N,qlogis(w0))
+    df[,w := plogis(w.noisey.odds)]
+    df[,w_pred:=as.integer(w > 0.5)]
+    (mean(df$x==df$w_pred))

     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")
     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")

-parser <- add_argument(parser, "--N", default=500, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=100, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=57, help='seed for the rng')

 parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')
 parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')

-parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
-parser <- add_argument(parser, "--gx_explained_variance", help='what proportion of the variance of x can be explained by g?', default=0.15)
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
+# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)

 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)

-
+parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1)

 args <- parse_args(parser)
 
 B0 <- 0
 args <- parse_args(parser)
 
 B0 <- 0

-Bxy <- 0.2
-Bgy <- -0.2
-Bgx <- 0.4
+Bxy <- 0.3
+Bzy <- -0.3
+Bzx <- args$Bzx

 
 

-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, Bgx, seed=args$seed, y_explained_variance = args$y_explained_variance, gx_explained_variance = args$gx_explained_variance, prediction_accuracy=args$prediction_accuracy)
+if (args$m < args$N){

 
 

-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'Bgx'=Bgx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'gx_explained_variance' = args$gx_explained_variance, "prediction_accuracy"=args$prediction_accuracy)
-outline <- run_simulation(df, result)
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance,  prediction_accuracy=args$prediction_accuracy)

 
 

-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
-    logdata <- read_feather(args$outfile)
-    logdata <- rbind(logdata,as.data.table(outline))
-} else {
-    logdata <- as.data.table(outline)
-}
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="")

 
 

-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)
+    outline <- run_simulation(df, result)
+    
+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

diff --git a/simulations/02_indep_differential.R b/simulations/02_indep_differential.R
index d4e091691c848a590e2873e16d94996b4a75a59a..7e2e428d5d2695498323c64a160cb866fcea1161 100644 (file)
--- a/simulations/02_indep_differential.R
+++ b/simulations/02_indep_differential.R
@@ -31,17 +31,17 @@ source("simulation_base.R")
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.

-simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){

     set.seed(seed)
     # make w and y dependent
     set.seed(seed)
     # make w and y dependent

-    g <- rbinom(N, 1, 0.5)
-    x <- rbinom(N, 1, 0.5)
+    z <- rbinom(N, 1, 0.5)
+    x <- rbinom(N, 1, Bzx * z + 0.5)

 
 

-    y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bgy*g,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
+    y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)

     y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
     y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))

-    y <- Bgy * g + Bxy * x + y.epsilon
-
-    df <- data.table(x=x,y=y,g=g)
+    y <- Bzy * z + Bxy * x + y.epsilon
+    
+    df <- data.table(x=x,y=y,z=z)

 
     if(m < N){
         df <- df[sample(nrow(df), m), x.obs := x]
 
     if(m < N){
         df <- df[sample(nrow(df), m), x.obs := x]


@@ -49,61 +49,117 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025,
         df <- df[, x.obs := x]
     }
 
         df <- df[, x.obs := x]
     }
 

-    df <- df[,w_pred:=x]
-
-    pg <- mean(g)
-    px <- mean(x)
-    accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
+    ## px <- mean(x)
+    ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)

 
 

-    # this works because of conditional probability
-    accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
-    accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
+    ## # this works because of conditional probability
+    ## accuracy_x0 <- prediction_accuracy / (px*(accuracy_imbalance_ratio) + (1-px))
+    ## accuracy_x1 <- accuracy_imbalance_ratio * accuracy_x0

 
 

-    dfg0 <- df[g==0]
-    ng0 <- nrow(dfg0)
-    dfg1 <- df[g==1]
-    ng1 <- nrow(dfg1)
+    ## x0 <- df[x==0]$x
+    ## x1 <- df[x==1]$x
+    ## nx1 <- nrow(df[x==1])
+    ## nx0 <- nrow(df[x==0])

 
 

-    dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
-    dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
+    ## yx0 <- df[x==0]$y
+    ## yx1 <- df[x==1]$y 
+ 
+    # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    ## acc.x0 <- plogis(0.5*scale(yx0) + qlogis(accuracy_x0))
+    ## acc.x1 <- plogis(1.5*scale(yx1) + qlogis(accuracy_x1))

 
 

-    df <- rbind(dfg0,dfg1)
+    ## w0x0 <- (1-x0)**2 + (-1)**(1-x0) * acc.x0
+    ## w0x1 <- (1-x1)**2 + (-1)**(1-x1) * acc.x1
+    pz <- mean(z)
+    accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)

 
 

-    w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
-    df <- df[,':='(w=w, w_pred = w_pred)]
+    # this works because of conditional probability
+    accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
+
+    z0x0 <- df[(z==0) & (x==0)]$x
+    z0x1 <- df[(z==0) & (x==1)]$x
+    z1x0 <- df[(z==1) & (x==0)]$x
+    z1x1 <- df[(z==1) & (x==1)]$x
+
+    yz0x0 <- df[(z==0) & (x==0)]$y
+    yz0x1 <- df[(z==0) & (x==1)]$y
+    yz1x0 <- df[(z==1) & (x==0)]$y
+    yz1x1 <- df[(z==1) & (x==1)]$y
+
+    nz0x0 <- nrow(df[(z==0) & (x==0)])
+    nz0x1 <- nrow(df[(z==0) & (x==1)])
+    nz1x0 <- nrow(df[(z==1) & (x==0)])
+    nz1x1 <- nrow(df[(z==1) & (x==1)])
+
+    yz1 <- df[z==1]$y 
+    yz1 <- df[z==1]$y 
+
+    # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
+    acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+    acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+    acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
+
+    w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
+    w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
+    w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
+    w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
+
+    ##perrorz0 <- w0z0*(pyz0)
+    ##perrorz1 <- w0z1*(pyz1)
+
+    w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
+    w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
+    w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
+    w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
+
+    df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
+    df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]    
+    df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]    
+    df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]    
+
+    df[,w_pred:=as.integer(w > 0.5)]
+    print(mean(df[z==0]$x == df[z==0]$w_pred))
+    print(mean(df[z==1]$x == df[z==1]$w_pred))
+    print(mean(df$w_pred == df$x))

     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")
     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")

-parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=432, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1400, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=50, help='seed for the rng')

 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
 parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
 parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)

+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+

 
 args <- parse_args(parser)
 
 B0 <- 0
 
 args <- parse_args(parser)
 
 B0 <- 0

-Bxy <- 0.2
-Bgy <- -0.2
-
-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
+Bxy <- 0.3
+Bzy <- args$Bzy

 
 

-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
+if(args$m < args$N){
+    df <- simulate_data(args$N, args$m, B0, Bxy, args$Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)

 
 

-outline <- run_simulation_depvar(df=df, result)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, error='')

 
 

+    outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x+z+y+x:y, truth_formula=x~z)
+    
+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }

 
 

-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
-    logdata <- read_feather(args$outfile)
-    logdata <- rbind(logdata,as.data.table(outline))
-} else {
-    logdata <- as.data.table(outline)
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)

 }
 }

-
-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)

diff --git a/simulations/03_depvar_differential.R b/simulations/03_depvar_differential.R
index d52afe7e3c501b399f2677f47c4768d54901a78e..872931f4c426a4680bb1aad03699668c90264c39 100644 (file)
--- a/simulations/03_depvar_differential.R
+++ b/simulations/03_depvar_differential.R
@@ -31,18 +31,18 @@ source("simulation_base.R")
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.

-simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){

     set.seed(seed)
     # make w and y dependent
     set.seed(seed)
     # make w and y dependent

-    g <- rbinom(N, 1, 0.5)
+    z <- rbinom(N, 1, 0.5)

     x <- rbinom(N, 1, 0.5)
 
     x <- rbinom(N, 1, 0.5)
 

-    ystar <- Bgy * g + Bxy * x
-    y <- rbinom(N,1,logistic(ystar))
+    ystar <- Bzy * z + Bxy * x
+    y <- rbinom(N,1,plogis(ystar))

 
 

-    # glm(y ~ x + g, family="binomial")
+    # glm(y ~ x + z, family="binomial")

 
 

-    df <- data.table(x=x,y=y,ystar=ystar,g=g)
+    df <- data.table(x=x,y=y,ystar=ystar,z=z)

 
     if(m < N){
         df <- df[sample(nrow(df), m), y.obs := y]
 
     if(m < N){
         df <- df[sample(nrow(df), m), y.obs := y]


@@ -52,36 +52,44 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, ac
 
     df <- df[,w_pred:=y]
 
 
     df <- df[,w_pred:=y]
 

-    pg <- mean(g)
+    pz <- mean(z)

 
     accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
 
     # this works because of conditional probability
 
     accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
 
     # this works because of conditional probability

-    accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
-    accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
+    accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0

 
 

-    dfg0 <- df[g==0]
-    ng0 <- nrow(dfg0)
-    dfg1 <- df[g==1]
-    ng1 <- nrow(dfg1)

 
 

-    dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
-    dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
+    yz0 <- df[z==0]$y
+    yz1 <- df[z==1]$y
+    nz1 <- nrow(df[z==1])
+    nz0 <- nrow(df[z==0])

 
 

-    df <- rbind(dfg0,dfg1)
+    acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0))
+    acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1))
+    
+    w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0
+    w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1
+    
+    w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0))
+    w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1))
+    df[z==0,w:=plogis(w0z0.noisy.odds)]
+    df[z==1,w:=plogis(w0z1.noisy.odds)]

 
 

-    wmod <- glm(y.obs ~ w_pred,data=df[!is.null(y.obs)],family=binomial(link='logit'))
-    w <- predict(wmod,df,type='response')
+    df[,w_pred:=as.integer(w > 0.5)]

 
 

-    df <- df[,':='(w=w)]
+    print(mean(df[y==0]$y == df[y==0]$w_pred))
+    print(mean(df[y==1]$y == df[y==1]$w_pred))
+    print(mean(df$w_pred == df$y))

 
     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")
 
     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")

-parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
-parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')

 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)


@@ -90,24 +98,26 @@ parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much
 args <- parse_args(parser)
 
 B0 <- 0
 args <- parse_args(parser)
 
 B0 <- 0

-Bxy <- 0.2
-Bgy <- -0.2
+Bxy <- 0.7
+Bzy <- -0.7

 
 

-df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
+if(args$m < args$N){
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)

 
 

-result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)

 
 

-outline <- run_simulation_depvar(df=df, result)
+    outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x)

 
 

+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)

 
 

-outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-if(file.exists(args$outfile)){
-    logdata <- read_feather(args$outfile)
-    logdata <- rbind(logdata,as.data.table(outline))
-} else {
-    logdata <- as.data.table(outline)
-}
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }

 
 

-print(outline)
-write_feather(logdata, args$outfile)
-unlock(outfile_lock)
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

diff --git a/simulations/Makefile b/simulations/Makefile
index 2b18fea4acb0a7bfb60a7281fbc0ebdd42c1a05b..dec7889db143e7738293e5825094d7ccc9cac2df 100644 (file)
--- a/simulations/Makefile
+++ b/simulations/Makefile
@@ -1,28 +1,42 @@
 
 SHELL=bash
 
 
 SHELL=bash
 

-Ns=[500,1000,10000]
-ms=[50, 100, 250, 500]
+Ns=[1000,3600,14400]
+ms=[75,150,300]

 seeds=[$(shell seq -s, 1 250)]
 seeds=[$(shell seq -s, 1 250)]

+explained_variances=[0.1]
+

 all:remembr.RDS
 
 all:remembr.RDS
 

-srun=srun -A comdata -p compute-bigmem --time=10:00:00 --mem 4G -c 1
+srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1
+
+
+joblists:example_1_jobs example_2_jobs example_3_jobs
+
+# test_true_z_jobs: test_true_z.R simulation_base.R
+#      grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb

 
 

-example_1_jobs: 01_two_covariates.R
-       grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"]}' --outfile example_1_jobs
+# test_true_z.feather: test_true_z_jobs 
+#      rm -f test_true_z.feather
+#      sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 test_true_z_jobs
+#      sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs

 
 

-example_1.feather: example_1_jobs
+
+example_1_jobs: 01_two_covariates.R simulation_base.R
+       grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs
+
+example_1.feather: example_1_jobs 

        rm -f example_1.feather
        rm -f example_1.feather

-       sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_1_jobs
-       sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
+       sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
+#      sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs

 
 

-example_2_jobs: example_2.R
-       grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"]}' --outfile example_2_jobs
+example_2_jobs: 02_indep_differential.R simulation_base.R
+       grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs

 
 

-example_2.feather: example_2_jobs
+example_2.feather: example_2_jobs 

        rm -f example_2.feather
        rm -f example_2.feather

-       sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_jobs
-       sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
+       sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
+#      sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs

 
 # example_2_B_jobs: example_2_B.R
 #      grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
 
 # example_2_B_jobs: example_2_B.R
 #      grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs


@@ -31,23 +45,24 @@ example_2.feather: example_2_jobs
 #      rm -f example_2_B.feather
 #      sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
 
 #      rm -f example_2_B.feather
 #      sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
 

-example_3_jobs: 03_depvar_differential.R
-       grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"]}' --outfile example_3_jobs
+example_3_jobs: 03_depvar_differential.R simulation_base.R
+       grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs

 
 example_3.feather: example_3_jobs
 
 example_3.feather: example_3_jobs

-       rm -f example_3.feather
-       sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_3_jobs
-       sbatch --wait --verbose --array=3001-6000 run_simulation.sbatch 0 example_3_jobs
+       rm -f example_3.feather 
+       sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l)  run_simulation.sbatch 0 example_3_jobs

 
 

-remembr.RDS:example_1.feather example_2.feather example_3.feather
+
+remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R
+       rm -f remembr.RDS

        ${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
        ${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
        ${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
 
 clean:
        rm *.feather
        ${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
        ${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
        ${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
 
 clean:
        rm *.feather

-       rm remembr.RDS
-
+       rm -f remembr.RDS
+       rm -f example_*_jobs

 #      sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
 
 # example_2_B_mecor_jobs:
 #      sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
 
 # example_2_B_mecor_jobs:

diff --git a/simulations/measerr_methods.R b/simulations/measerr_methods.R
new file mode 100644 (file)
index 0000000..ab87d71
--- /dev/null
+++ b/simulations/measerr_methods.R
@@ -0,0 +1,227 @@
+library(formula.tools)
+library(matrixStats)
+
+## df: dataframe to model
+## outcome_formula: formula for y | x, z
+## outcome_family: family for y | x, z
+## proxy_formula: formula for w | x, z, y 
+## proxy_family: family for w | x, z, y
+## truth_formula: formula for x | z
+## truth_family: family for x | z
+
+### ideal formulas for example 1
+# test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
+
+### ideal formulas for example 2
+# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
+
+
+## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
+
+    nll <- function(params){
+        df.obs <- model.frame(outcome_formula, df)
+        proxy.variable <- all.vars(proxy_formula)[1]
+        proxy.model.matrix <- model.matrix(proxy_formula, df)
+        response.var <- all.vars(outcome_formula)[1]
+        y.obs <- with(df.obs,eval(parse(text=response.var)))
+        outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
+
+        param.idx <- 1
+        n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+        outcome.params <- params[param.idx:n.outcome.model.covars]
+        param.idx <- param.idx + n.outcome.model.covars
+
+        if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
+            ll.y.obs <- vector(mode='numeric', length=length(y.obs))
+            ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
+            ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
+        }
+
+        df.obs <- model.frame(proxy_formula,df)
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+
+        param.idx <- param.idx + n.proxy.model.covars
+        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
+
+        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+            ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
+            ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
+            ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+        }
+
+        ll.obs <- sum(ll.y.obs + ll.w.obs)
+
+        df.unobs <- df[is.na(df[[response.var]])]
+        df.unobs.y1 <- copy(df.unobs)
+        df.unobs.y1[[response.var]] <- 1
+        df.unobs.y0 <- copy(df.unobs)
+        df.unobs.y0[[response.var]] <- 1
+        
+        ## integrate out y
+        outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
+
+        if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
+            ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
+            ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
+            ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
+            ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
+        }
+
+        proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
+        proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
+        proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
+
+        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+            ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
+            ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
+            ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
+            ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+
+            ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
+            ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+        }
+
+        ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
+        ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
+        ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
+        ll <- ll.unobs + ll.obs
+        return(-ll)
+    }
+    
+    params <- colnames(model.matrix(outcome_formula,df))
+    lower <- rep(-Inf, length(params))
+    proxy.params <- colnames(model.matrix(proxy_formula, df))
+    params <- c(params, paste0('proxy_',proxy.params))
+    lower <- c(lower, rep(-Inf, length(proxy.params)))
+    start <- rep(0.1,length(params))
+    names(start) <- params
+    
+    fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    return(fit)
+}
+
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
+
+    measrr_mle_nll <- function(params){
+        df.obs <- model.frame(outcome_formula, df)
+        
+        proxy.variable <- all.vars(proxy_formula)[1]
+        proxy.model.matrix <- model.matrix(proxy_formula, df)
+
+        response.var <- all.vars(outcome_formula)[1]
+        y.obs <- with(df.obs,eval(parse(text=response.var)))
+        
+        outcome.model.matrix <- model.matrix(outcome_formula, df)
+
+        param.idx <- 1
+        n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+        outcome.params <- params[param.idx:n.outcome.model.covars]
+        param.idx <- param.idx + n.outcome.model.covars
+
+        ## likelihood for the fully observed data 
+        if(outcome_family$family == "gaussian"){
+            sigma.y <- params[param.idx]
+            param.idx <- param.idx + 1
+            ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
+        }
+        
+        df.obs <- model.frame(proxy_formula,df)
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+        param.idx <- param.idx + n.proxy.model.covars
+        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
+
+        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+            ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
+            ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
+            ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+        }
+
+        df.obs <- model.frame(truth_formula, df)
+        truth.variable <- all.vars(truth_formula)[1]
+        truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
+        truth.model.matrix <- model.matrix(truth_formula,df)
+        n.truth.model.covars <- dim(truth.model.matrix)[2]
+        
+        truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+        if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
+            ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
+            ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
+            ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
+        }
+        
+        ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
+
+        ## likelihood for the predicted data
+        ## integrate out the "truth" variable. 
+        
+        if(truth_family$family=='binomial'){
+            df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
+            df.unobs.x1 <- copy(df.unobs)
+            df.unobs.x1[,'x'] <- 1
+            df.unobs.x0 <- copy(df.unobs)
+            df.unobs.x0[,'x'] <- 0
+            outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
+            
+            outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
+            outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
+            if(outcome_family$family=="gaussian"){
+            ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+            ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+            }
+
+            if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
+
+                proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+                proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+                proxy.unobs <- df.unobs[[proxy.variable]]
+                ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
+                ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
+
+                ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
+                ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
+
+                ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
+                ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
+            }
+
+            if(truth_family$link=='logit'){
+                truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
+                ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
+                ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
+            }
+        }
+
+        ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
+        ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
+        ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
+        return(-(ll.unobs + ll.obs))
+    }
+    
+    outcome.params <- colnames(model.matrix(outcome_formula,df))
+    lower <- rep(-Inf, length(outcome.params))
+
+    if(outcome_family$family=='gaussian'){
+        params <- c(outcome.params, 'sigma_y')
+        lower <- c(lower, 0.00001)
+    } else {
+        params <- outcome.params
+    }
+    
+    proxy.params <- colnames(model.matrix(proxy_formula, df))
+    params <- c(params, paste0('proxy_',proxy.params))
+    lower <- c(lower, rep(-Inf, length(proxy.params)))
+
+    truth.params <- colnames(model.matrix(truth_formula, df))
+    params <- c(params, paste0('truth_', truth.params))
+    lower <- c(lower, rep(-Inf, length(truth.params)))
+    start <- rep(0.1,length(params))
+    names(start) <- params
+    
+    fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+
+    return(fit)
+}

diff --git a/simulations/plot_dv_example.R b/simulations/plot_dv_example.R
index 961bc879c3cbbfa111a71785af70fca606c82adb..f69ed6c012f9f1c8576bbc0a75d92e522293ffa1 100644 (file)
--- a/simulations/plot_dv_example.R
+++ b/simulations/plot_dv_example.R
@@ -10,204 +10,73 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 

-build_plot_dataset <- function(df){
-    x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
-    x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
-                             zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
-                             bias = Bxy - Bxy.est.naive,
-                             Bxy.est.naive = Bxy.est.naive,
-                             sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
-
-    x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
-                               mean.bias = mean(bias),
-                               mean.est = mean(Bxy.est.naive),
-                               var.est = var(Bxy.est.naive),
-                               N.sims = .N,
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               variable='x',
-                               method='Naive'
-                               ),
-                            by=c('N','m')]
-    

 
 

-    g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
-    g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
-                             zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
-                             bias = Bgy - Bgy.est.naive,
-                             Bgy.est.naive = Bgy.est.naive,
-                             sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
-
-    g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
-                               mean.bias = mean(bias),
-                               mean.est = mean(Bgy.est.naive),
-                               var.est = var(Bgy.est.naive),
-                               N.sims = .N,
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               variable='g',
-                               method='Naive'
-                               ),
-                            by=c('N','m')]
-    

 
 

-    x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
-    x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
-                                   zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
-                                   bias = Bxy - Bxy.est.feasible,
-                                   Bxy.est.feasible = Bxy.est.feasible,
-                                   sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
-
-    x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
-                                     mean.bias = mean(bias),
-                                     mean.est = mean(Bxy.est.feasible),
-                                     var.est = var(Bxy.est.feasible),
-                                     N.sims = .N,
-                                     p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                     variable='x',
-                                     method='Feasible'
-                                     ),
-                                  by=c('N','m')]
-    
+summarize.estimator <- function(df, suffix='naive', coefname='x'){

 
 

-    g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
-    g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
-                                   zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
-                                   bias = Bgy - Bgy.est.feasible,
-                                   Bgy.est.feasible = Bgy.est.feasible,
-                                   sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
-
-    g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
-                                     mean.bias = mean(bias),
-                                     mean.est = mean(Bgy.est.feasible),
-                                     var.est = var(Bgy.est.feasible),
-                                     N.sims = .N,
-                                     p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                     variable='g',
-                                     method='Feasible'
-                                     ),
-                                  by=c('N','m')]
+    part <- df[,c('N',
+                  'm',
+                  'Bxy',
+                  paste0('B',coefname,'y.est.',suffix),
+                  paste0('B',coefname,'y.ci.lower.',suffix),
+                  paste0('B',coefname,'y.ci.upper.',suffix),
+                  'y_explained_variance',
+                  'Bzy',
+                  'accuracy_imbalance_difference'
+                  ),
+               with=FALSE]

     
     

+    true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+    zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+    bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+    sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+    part <- part[,':='(true.in.ci = true.in.ci,
+                       zero.in.ci = zero.in.ci,
+                       bias=bias,
+                       sign.correct =sign.correct)]
+
+    part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+                          mean.bias = mean(bias),
+                          mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+                          var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+                          est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
+                          est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
+                          N.sims = .N,
+                          p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+                          variable=coefname,
+                          method=suffix
+                          ),
+                      by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance')
+                      ]
+    
+    return(part.plot)
+}
+

 
 

+build_plot_dataset <- function(df){
+
+    x.true <- summarize.estimator(df, 'true','x')
+    z.true <- summarize.estimator(df, 'true','z')

 
 

-    x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
-
-    x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
-                                         zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
-                                         bias = Bxy.est.true - Bxy.est.amelia.full,
-                                         sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
-
-    x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                           mean.bias = mean(bias),
-                                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                           mean.est = mean(Bxy.est.amelia.full),
-                                           var.est = var(Bxy.est.amelia.full),
-                                           N.sims = .N,
-                                           variable='x',
-                                           method='Multiple imputation'
-                                           ),
-                                        by=c('N','m')]
-
-
-    g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
-    g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
-                                         zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
-                                         bias =  Bgy.est.amelia.full - Bgy.est.true,
-                                         sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
-
-    g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                           mean.bias = mean(bias),
-                                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                           mean.est = mean(Bgy.est.amelia.full),
-                                           var.est = var(Bgy.est.amelia.full),
-                                           N.sims = .N,
-                                           variable='g',
-                                           method='Multiple imputation'
-                                           ),
-                                        by=c('N','m')]
-
-    x.mle <- df[,.(N,m, Bxy.est.true, Bxy.est.mle, Bxy.ci.lower.mle, Bxy.ci.upper.mle)]
-
-    x.mle <- x.mle[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.mle) & (Bxy.est.true <= Bxy.ci.upper.mle),
-                                         zero.in.ci = (0 >= Bxy.ci.lower.mle) & (0 <= Bxy.ci.upper.mle),
-                                         bias =  Bxy.est.mle - Bxy.est.true,
-                                         sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mle))]
-
-    x.mle.plot <- x.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                   mean.bias = mean(bias),
-                                   p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                   mean.est = mean(Bxy.est.mle),
-                                   var.est = var(Bxy.est.mle),
-                                   N.sims = .N,
-                                   variable='x',
-                                   method='Maximum Likelihood'
-                                   ),
-                                by=c('N','m')]
-
-
-
-    g.mle <- df[,.(N,m, Bgy.est.true, Bgy.est.mle, Bgy.ci.lower.mle, Bgy.ci.upper.mle)]
-
-    g.mle <- g.mle[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mle) & (Bgy.est.true <= Bgy.ci.upper.mle),
-                                         zero.in.ci = (0 >= Bgy.ci.lower.mle) & (0 <= Bgy.ci.upper.mle),
-                                         bias =  Bgy.est.mle - Bgy.est.true,
-                                         sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mle))]
-
-    g.mle.plot <- g.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                   mean.bias = mean(bias),
-                                   p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                   mean.est = mean(Bgy.est.mle),
-                                   var.est = var(Bgy.est.mle),
-                                   N.sims = .N,
-                                   variable='g',
-                                   method='Maximum Likelihood'
-                                   ),
-                                by=c('N','m')]
-
-
-
-
-    x.pseudo <- df[,.(N,m, Bxy.est.true, Bxy.est.pseudo, Bxy.ci.lower.pseudo, Bxy.ci.upper.pseudo)]
-
-    x.pseudo <- x.pseudo[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.pseudo) & (Bxy.est.true <= Bxy.ci.upper.pseudo),
-                                         zero.in.ci = (0 >= Bxy.ci.lower.pseudo) & (0 <= Bxy.ci.upper.pseudo),
-                                         bias =  Bxy.est.pseudo - Bxy.est.true,
-                                         sign.correct = sign(Bxy.est.true) == sign(Bxy.est.pseudo))]
-
-    x.pseudo.plot <- x.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                   mean.bias = mean(bias),
-                                   p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                   mean.est = mean(Bxy.est.pseudo),
-                                   var.est = var(Bxy.est.pseudo),
-                                   N.sims = .N,
-                                   variable='x',
-                                   method='Pseudo Likelihood'
-                                   ),
-                                by=c('N','m')]
-
-
-
-    g.pseudo <- df[,.(N,m, Bgy.est.true, Bgy.est.pseudo, Bgy.ci.lower.pseudo, Bgy.ci.upper.pseudo)]
-
-    g.pseudo <- g.pseudo[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.pseudo) & (Bgy.est.true <= Bgy.ci.upper.pseudo),
-                                         zero.in.ci = (0 >= Bgy.ci.lower.pseudo) & (0 <= Bgy.ci.upper.pseudo),
-                                         bias =  Bgy.est.pseudo - Bgy.est.true,
-                                         sign.correct = sign(Bgy.est.true) == sign(Bgy.est.pseudo))]
-
-    g.pseudo.plot <- g.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                   mean.bias = mean(bias),
-                                   p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                   mean.est = mean(Bgy.est.pseudo),
-                                   var.est = var(Bgy.est.pseudo),
-                                   N.sims = .N,
-                                   variable='g',
-                                   method='Pseudo Likelihood'
-                                   ),
-                                by=c('N','m')]
+    x.naive <- summarize.estimator(df, 'naive','x')
+    z.naive <- summarize.estimator(df, 'naive','z')
+    
+    x.feasible <- summarize.estimator(df, 'feasible','x')
+    z.feasible <- summarize.estimator(df, 'feasible','z')
+    
+    x.amelia.full <- summarize.estimator(df, 'amelia.full','x')
+    z.amelia.full <- summarize.estimator(df, 'amelia.full','z')

 
 

+    x.mle <- summarize.estimator(df, 'mle','x')
+    z.mle <- summarize.estimator(df, 'mle','z')

 
 

+    x.zhang <- summarize.estimator(df, 'zhang','x')
+    z.zhang <- summarize.estimator(df, 'zhang','z')

     
     accuracy <- df[,mean(accuracy)]
 
     
     accuracy <- df[,mean(accuracy)]
 

-    plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mle.plot, g.mle.plot, x.pseudo.plot, g.pseudo.plot, x.feasible.plot, g.feasible.plot),use.names=T)
+    plot.df <- rbindlist(list(x.true, z.true, x.naive,z.naive,x.amelia.full,z.amelia.full,x.mle, z.mle, x.zhang, z.zhang, x.feasible, z.feasible),use.names=T)

 
     plot.df[,accuracy := accuracy]
 
 
     plot.df[,accuracy := accuracy]
 


@@ -219,15 +88,22 @@ build_plot_dataset <- function(df){
 
 df <- read_feather(args$infile)
 plot.df <- build_plot_dataset(df)
 
 df <- read_feather(args$infile)
 plot.df <- build_plot_dataset(df)

+

 remember(plot.df,args$name)
 
 
 ## df[gmm.ER_pval<0.05]
 remember(plot.df,args$name)
 
 
 ## df[gmm.ER_pval<0.05]

+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]

 
 
 
 

+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & !is.na(p.true.in.ci) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=-0.05),linetype=2)

 
 

-
-
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)

 ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
 
 ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 
 ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
 
 ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 

diff --git a/simulations/plot_example.R b/simulations/plot_example.R
index 1a4be9b2ab6676ea090bb6030ef96a5c5282f7f1..ebfd3a9c9a5be8cd2da9dce6d11dc1ce8aa9c70e 100644 (file)
--- a/simulations/plot_example.R
+++ b/simulations/plot_example.R
@@ -10,267 +10,172 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 

-build_plot_dataset <- function(df){
-    x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
-    x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
-                             zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
-                             bias = Bxy - Bxy.est.naive,
-                             Bxy.est.naive = Bxy.est.naive,
-                             sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
-
-    x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
-                               mean.bias = mean(bias),
-                               mean.est = mean(Bxy.est.naive),
-                               var.est = var(Bxy.est.naive),
-                               N.sims = .N,
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               variable='x',
-                               method='Naive'
-                               ),
-                            by=c('N','m')]
+summarize.estimator <- function(df, suffix='naive', coefname='x'){
+
+    part <- df[,c('N',
+                  'm',
+                  'Bxy',
+                  paste0('B',coefname,'y.est.',suffix),
+                  paste0('B',coefname,'y.ci.lower.',suffix),
+                  paste0('B',coefname,'y.ci.upper.',suffix),
+                  'y_explained_variance',
+                  'Bzx',
+                  'Bzy',
+                  'accuracy_imbalance_difference'
+                  ),
+               with=FALSE]
+    
+    true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+    zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+    bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+    sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+    part <- part[,':='(true.in.ci = true.in.ci,
+                       zero.in.ci = zero.in.ci,
+                       bias=bias,
+                       sign.correct =sign.correct)]
+
+    part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+                          mean.bias = mean(bias),
+                          mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+                          var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+                          est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
+                          est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
+                          N.sims = .N,
+                          p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+                          variable=coefname,
+                          method=suffix
+                          ),
+                      by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
+                      ]

     
     

+    return(part.plot)
+}

 
 

-    g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
-    g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
-                             zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
-                             bias = Bgy - Bgy.est.naive,
-                             Bgy.est.naive = Bgy.est.naive,
-                             sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
-
-    g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
-                               mean.bias = mean(bias),
-                               mean.est = mean(Bgy.est.naive),
-                               var.est = var(Bgy.est.naive),
-                               N.sims = .N,
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               variable='g',
-                               method='Naive'
-                               ),
-                            by=c('N','m')]
+build_plot_dataset <- function(df){

     
     

+    x.true <-  summarize.estimator(df, 'true','x')
+
+    z.true <-  summarize.estimator(df, 'true','z')

 
 

-    x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
-    x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
-                                   zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
-                                   bias = Bxy - Bxy.est.feasible,
-                                   Bxy.est.feasible = Bxy.est.feasible,
-                                   sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
-
-    x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
-                                     mean.bias = mean(bias),
-                                     mean.est = mean(Bxy.est.feasible),
-                                     var.est = var(Bxy.est.feasible),
-                                     N.sims = .N,
-                                     p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                     variable='x',
-                                     method='Feasible'
-                                     ),
-                                  by=c('N','m')]
+    x.naive <- summarize.estimator(df, 'naive','x')

     
     

+    z.naive <- summarize.estimator(df,'naive','z')

 
 

-    g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
-    g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
-                                   zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
-                                   bias = Bgy - Bgy.est.feasible,
-                                   Bgy.est.feasible = Bgy.est.feasible,
-                                   sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
-
-    g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
-                                     mean.bias = mean(bias),
-                                     mean.est = mean(Bgy.est.feasible),
-                                     var.est = var(Bgy.est.feasible),
-                                     N.sims = .N,
-                                     p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                     variable='g',
-                                     method='Feasible'
-                                     ),
-                                  by=c('N','m')]
+    x.feasible <- summarize.estimator(df, 'feasible', 'x')
+
+    z.feasible <- summarize.estimator(df, 'feasible', 'z')
+
+    x.amelia.full <- summarize.estimator(df, 'amelia.full', 'x')
+
+    z.amelia.full <- summarize.estimator(df, 'amelia.full', 'z')

     
     

+    x.mecor <- summarize.estimator(df, 'mecor', 'x')

 
 

+    z.mecor <- summarize.estimator(df, 'mecor', 'z')

 
 

-    x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
-
-    x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
-                                         zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
-                                         bias = Bxy.est.true - Bxy.est.amelia.full,
-                                         sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
-
-    x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                           mean.bias = mean(bias),
-                                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                           mean.est = mean(Bxy.est.amelia.full),
-                                           var.est = var(Bxy.est.amelia.full),
-                                           N.sims = .N,
-                                           variable='x',
-                                           method='Multiple imputation'
-                                           ),
-                                        by=c('N','m')]
-
-
-    g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
-    g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
-                                         zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
-                                         bias =  Bgy.est.amelia.full - Bgy.est.true,
-                                         sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
-
-    g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                                           mean.bias = mean(bias),
-                                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                                           mean.est = mean(Bgy.est.amelia.full),
-                                           var.est = var(Bgy.est.amelia.full),
-                                           N.sims = .N,
-                                           variable='g',
-                                           method='Multiple imputation'
-                                           ),
-                                        by=c('N','m')]
-
-    ## x.amelia.nok <- df[,.(N, m, Bxy.est.true, Bxy.est.amelia.nok, Bxy.ci.lower.amelia.nok, Bxy.ci.upper.amelia.nok)]
-    ## x.amelia.nok <- x.amelia.nok[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.nok) & (Bxy.est.true <= Bxy.ci.upper.amelia.nok),
-    ##                                    zero.in.ci = (0 >= Bxy.ci.lower.amelia.nok) & (0 <= Bxy.ci.upper.amelia.nok),
-    ##                                    bias =  Bxy.est.amelia.nok - Bxy.est.true,
-    ##                                    sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.nok))]
-
-    ## x.amelia.nok.plot <- x.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-    ##                                      mean.bias = mean(bias),
-    ##                                      p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-    ##                                      mean.est = mean(Bxy.est.amelia.nok),
-    ##                                      var.est = var(Bxy.est.amelia.nok),
-    ##                                      N.sims = .N,
-    ##                                      variable='x',
-    ##                                      method='Multiple imputation (Classifier features unobserved)'
-    ##                                      ),
-    ##                                   by=c('N','m')]
-
-    ## g.amelia.nok <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.nok, Bgy.ci.lower.amelia.nok, Bgy.ci.upper.amelia.nok)]
-    ## g.amelia.nok <- g.amelia.nok[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.nok) & (Bgy.est.true <= Bgy.ci.upper.amelia.nok),
-    ##                                    zero.in.ci = (0 >= Bgy.ci.lower.amelia.nok) & (0 <= Bgy.ci.upper.amelia.nok),
-    ##                                    bias =  Bgy.est.amelia.nok - Bgy.est.true,
-    ##                                    sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.nok))]
-
-    ## g.amelia.nok.plot <- g.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-    ##                                      mean.bias = mean(bias),
-    ##                                      p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-    ##                                      mean.est = mean(Bgy.est.amelia.nok),
-    ##                                      var.est = var(Bgy.est.amelia.nok),
-    ##                                      N.sims = .N,
-    ##                                      variable='g',
-    ##                                      method="Multiple imputation (Classifier features unobserved)"
-    ##                                      ),
-    ##                                   by=c('N','m')]
-
-
-    x.mecor <- df[,.(N,m,Bxy.est.true, Bxy.est.mecor,Bxy.lower.mecor, Bxy.upper.mecor)]
-
-    x.mecor <- x.mecor[,':='(true.in.ci = (Bxy.est.true >= Bxy.lower.mecor) & (Bxy.est.true <= Bxy.upper.mecor),
-                             zero.in.ci = (0 >= Bxy.lower.mecor) & (0 <= Bxy.upper.mecor),
-                             bias =  Bxy.est.mecor - Bxy.est.true,
-                             sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mecor))]
-
-    x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                               mean.bias = mean(bias),
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               mean.est = mean(Bxy.est.mecor),
-                               var.est = var(Bxy.est.mecor),
-                               N.sims = .N,
-                               variable='x',
-                               method='Regression Calibration'
-                               ),
-                            by=c('N','m')]
-
-    g.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.lower.mecor, Bgy.upper.mecor)]
-
-    g.mecor <- g.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.lower.mecor) & (Bgy.est.true <= Bgy.upper.mecor),
-                             zero.in.ci = (0 >= Bgy.lower.mecor) & (0 <= Bgy.upper.mecor),
-                             bias =  Bgy.est.mecor - Bgy.est.true,
-                             sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
-
-    g.mecor.plot <- g.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                               mean.bias = mean(bias),
-                               p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                               mean.est = mean(Bgy.est.mecor),
-                               var.est = var(Bgy.est.mecor),
-                               N.sims = .N,
-                               variable='g',
-                               method='Regression Calibration'
-                               ),
-                            by=c('N','m')]
-
-    ## x.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.ci.lower.mecor, Bgy.ci.upper.mecor)]
-
-    ## x.mecor <- x.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mecor) & (Bgy.est.true <= Bgy.ci.upper.mecor),
-    ##                                      zero.in.ci = (0 >= Bgy.ci.lower.mecor) & (0 <= Bgy.ci.upper.mecor),
-    ##                                      bias =  Bgy.est.mecor - Bgy.est.true,
-    ##                                      sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
-
-    ## x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-    ##                                        mean.bias = mean(bias),
-    ##                                      p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-    ##                                      variable='g',
-    ##                                      method='Regression Calibration'
-    ##                                      ),
-    ##                                     by=c('N','m')]
-
-
-    x.gmm <- df[,.(N,m,Bxy.est.true, Bxy.est.gmm,Bxy.ci.lower.gmm, Bxy.ci.upper.gmm)]
-    x.gmm <- x.gmm[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.gmm) & (Bxy.est.true <= Bxy.ci.upper.gmm),
-                         zero.in.ci = (0 >= Bxy.ci.lower.gmm) & (0 <= Bxy.ci.upper.gmm),
-                         bias =  Bxy.est.gmm - Bxy.est.true,
-                         sign.correct = sign(Bxy.est.true) == sign(Bxy.est.gmm))]
-
-    x.gmm.plot <- x.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                           mean.bias = mean(bias),
-                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                           mean.est = mean(Bxy.est.gmm),
-
-                           var.est = var(Bxy.est.gmm),
-                           N.sims = .N,
-                           variable='x',
-                           method='2SLS+gmm'
-                           ),
-                        by=c('N','m')]
-
-    g.gmm <- df[,.(N,m,Bgy.est.true, Bgy.est.gmm,Bgy.ci.lower.gmm, Bgy.ci.upper.gmm)]
-    g.gmm <- g.gmm[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.gmm) & (Bgy.est.true <= Bgy.ci.upper.gmm),
-                         zero.in.ci = (0 >= Bgy.ci.lower.gmm) & (0 <= Bgy.ci.upper.gmm),
-                         bias =  Bgy.est.gmm - Bgy.est.true,
-                         sign.correct = sign(Bgy.est.true) == sign(Bgy.est.gmm))]
-
-    g.gmm.plot <- g.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
-                           mean.bias = mean(bias),
-                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                           mean.est = mean(Bgy.est.gmm),
-                           var.est = var(Bgy.est.gmm),
-                           N.sims = .N,
-                           variable='g',
-                           method='2SLS+gmm'
-                           ),
-                        by=c('N','m')]
+    x.mecor <- summarize.estimator(df, 'mecor', 'x')

 
 

-    accuracy <- df[,mean(accuracy)]
+    z.mecor <- summarize.estimator(df, 'mecor', 'z')

 
 

-    plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mecor.plot, g.mecor.plot, x.gmm.plot, g.gmm.plot, x.feasible.plot, g.feasible.plot),use.names=T)
+    x.mle <- summarize.estimator(df, 'mle', 'x')

 
 

-    plot.df[,accuracy := accuracy]
+    z.mle <- summarize.estimator(df, 'mle', 'z')
+    
+    x.zhang <- summarize.estimator(df, 'zhang', 'x')

 
 

-    plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+    z.zhang <- summarize.estimator(df, 'zhang', 'z')
+
+    x.gmm <- summarize.estimator(df, 'gmm', 'x')

 
 

+    z.gmm <- summarize.estimator(df, 'gmm', 'z')
+
+    accuracy <- df[,mean(accuracy)]
+    plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.mecor, z.mecor, x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T)
+    plot.df[,accuracy := accuracy]
+    plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]

     return(plot.df)
 }
 
 
     return(plot.df)
 }
 
 

-df <- read_feather(args$infile)
-plot.df <- build_plot_dataset(df)
+plot.df <- read_feather(args$infile)
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(plot.df)))
+    plot.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+    plot.df[,accuracy_imbalance_difference:=NA]
+
+unique(plot.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(plot.df)
+

 remember(plot.df,args$name)
 
 remember(plot.df,args$name)
 

+#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
+
+## ## ## df[gmm.ER_pval<0.05]
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle),
+##                        var.est = var(Bxy.est.mle),
+##                        N.sims = .N,
+##                        variable='z',
+##                        method='Bespoke MLE'
+##                        ),
+##                     by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')]
+
+## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]

 
 

-## df[gmm.ER_pval<0.05]
+## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle),
+##                        var.est = var(Bzy.est.mle),
+##                        N.sims = .N,
+##                        variable='z',
+##                        method='Bespoke MLE'
+##                        ),
+##                     by=c("N","m",'y_explained_variance','Bzx')]

 
 

+## plot.df <- z.mle.plot
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]

 
 

+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=0.2),linetype=2)

 
 

+## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)

 
 
 
 

-## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
+## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))

 
 

-## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 
+## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 

 
 

-## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 
+## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 

diff --git a/simulations/run_simulation.sbatch b/simulations/run_simulation.sbatch
index 835f39b9ad6a2070dd7665f9b783bc5fba9fc985..54f56bef90625764e39be20000a4b35e98d6c501 100644 (file)
--- a/simulations/run_simulation.sbatch
+++ b/simulations/run_simulation.sbatch
@@ -6,14 +6,14 @@
 ## Resources
 #SBATCH --nodes=1    
 ## Walltime (12 hours)
 ## Resources
 #SBATCH --nodes=1    
 ## Walltime (12 hours)

-#SBATCH --time=24:00:00
+#SBATCH --time=1:00:00

 ## Memory per node
 #SBATCH --mem=8G
 #SBATCH --cpus-per-task=1
 #SBATCH --ntasks-per-node=1
 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
 #SBATCH --output=simulation_jobs/%A_%a.out
 ## Memory per node
 #SBATCH --mem=8G
 #SBATCH --cpus-per-task=1
 #SBATCH --ntasks-per-node=1
 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
 #SBATCH --output=simulation_jobs/%A_%a.out

-#SBATCH --error=simulation_jobs/%A_%a.out
+#SBATCH --error=simulation_jobs/%A_%a.err

 
 TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
 TASK_CALL=$(sed -n ${TASK_NUM}p $2)
 
 TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
 TASK_CALL=$(sed -n ${TASK_NUM}p $2)

diff --git a/simulations/simulation_base.R b/simulations/simulation_base.R
index a73ed79945c7e754ed0b122a385592839152b36d..0f03276c432f257ede87dcc8c0d78fca6834557b 100644 (file)
--- a/simulations/simulation_base.R
+++ b/simulations/simulation_base.R
@@ -4,207 +4,324 @@ options(amelia.parallel="no",
         amelia.ncpus=1)
 library(Amelia)
 library(Zelig)
         amelia.ncpus=1)
 library(Amelia)
 library(Zelig)

-library(stats4)
+library(bbmle)
+library(matrixStats) # for numerically stable logsumexps

 
 

+source("measerr_methods.R") ## for my more generic function.

 
 ## This uses the pseudolikelihood approach from Carroll page 349.
 ## assumes MAR
 ## assumes differential error, but that only depends on Y
 ## inefficient, because pseudolikelihood
 
 ## This uses the pseudolikelihood approach from Carroll page 349.
 ## assumes MAR
 ## assumes differential error, but that only depends on Y
 ## inefficient, because pseudolikelihood

-logistic.correction.pseudo <- function(df){
+    
+## This uses the pseudo-likelihood approach from Carroll page 346.
+my.pseudo.mle <- function(df){

     p1.est <- mean(df[w_pred==1]$y.obs==1,na.rm=T)
     p0.est <- mean(df[w_pred==0]$y.obs==0,na.rm=T)
     
     p1.est <- mean(df[w_pred==1]$y.obs==1,na.rm=T)
     p0.est <- mean(df[w_pred==0]$y.obs==0,na.rm=T)
     

-    nll <- function(B0, Bxy, Bgy){
-        probs <- (1 - p0.est) + (p1.est + p0.est - 1)*plogis(B0 + Bxy * df$x + Bgy * df$g)
+    nll <- function(B0, Bxy, Bzy){

 
 

-        part1 = sum(log(probs[df$w_pred == 1]))
-        part2 = sum(log(1-probs[df$w_pred == 0]))
+        pw <- vector(mode='numeric',length=nrow(df))
+        dfw1 <- df[w_pred==1]
+        dfw0 <- df[w_pred==0]
+        pw[df$w_pred==1] <- plogis(B0 + Bxy * dfw1$x + Bzy * dfw1$z, log=T)
+        pw[df$w_pred==0] <- plogis(B0 + Bxy * dfw0$x + Bzy * dfw0$z, lower.tail=FALSE, log=T)

         
         

-        return(-1*(part1 + part2))
+        probs <- colLogSumExps(rbind(log(1 - p0.est), log(p1.est + p0.est - 1) + pw))
+        return(-1*sum(probs))

     }
     
     }
     

-    mlefit <- stats4::mle(minuslogl = nll, start = list(B0=0, Bxy=0.0, Bgy=0.0))
+    mlefit <- mle2(minuslogl = nll, start = list(B0=0.0, Bxy=0.0, Bzy=0.0), control=list(maxit=1e6),method='L-BFGS-B')

     return(mlefit)
 
 }
 
     return(mlefit)
 
 }
 

+
+## model from Zhang's arxiv paper, with predictions for y
+## Zhang got this model from Hausman 1998
+### I think this is actually eqivalent to the pseudo.mle method
+zhang.mle.iv <- function(df){
+    nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){
+    df.obs <- df[!is.na(x.obs)]
+    df.unobs <- df[is.na(x.obs)]
+
+    ## fpr = 1 - TNR
+    ### Problem: accounting for uncertainty in ppv / npv
+    
+    ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T))
+    ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T))
+
+    ## fnr = 1 - TPR
+    ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
+    ll <- sum(ll.y.obs)
+    ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs)
+
+    # unobserved case; integrate out x
+    ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
+    ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
+
+    ## case x == 1
+    lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0))
+    
+    ## case x == 0
+    lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
+
+    lls <- colLogSumExps(rbind(lls.x.1, lls.x.0))
+    ll <- ll + sum(lls)
+    return(-ll)
+    }    
+    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001),
+                   upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B')
+    return(mlefit)
+}
+
+## this is equivalent to the pseudo-liklihood model from Carolla
+zhang.mle.dv <- function(df){
+
+    nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
+    df.obs <- df[!is.na(y.obs)]
+
+    ## fpr = 1 - TNR
+    ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
+    ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
+
+    # observed case
+    ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
+    ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
+    ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
+
+    ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
+
+    # unobserved case; integrate out y
+    ## case y = 1
+    ll.y.1 <- vector(mode='numeric', length=nrow(df))
+    pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
+    ## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
+    lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
+    
+    ## case y = 0
+    ll.y.0 <- vector(mode='numeric', length=nrow(df))
+    pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
+
+    ## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
+    lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
+
+    lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
+    ll <- ll + sum(lls)
+    return(-ll)
+    }    
+    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
+                   upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
+    return(mlefit)
+}
+

 ## This uses the likelihood approach from Carroll page 353.
 ## assumes that we have a good measurement error model
 ## This uses the likelihood approach from Carroll page 353.
 ## assumes that we have a good measurement error model

-logistic.correction.liklihood <- function(df){
+my.mle <- function(df){

     
     ## liklihood for observed responses
     
     ## liklihood for observed responses

-    nll <- function(B0, Bxy, Bgy, gamma0, gamma_y, gamma_g){
+    nll <- function(B0, Bxy, Bzy, gamma0, gamma_y, gamma_z, gamma_yz){

         df.obs <- df[!is.na(y.obs)]
         df.obs <- df[!is.na(y.obs)]

-        p.y.obs <- plogis(B0 + Bxy * df.obs$x + Bgy*df.obs$g)
-        p.y.obs[df.obs$y==0] <- 1-p.y.obs[df.obs$y==0]
-        p.s.obs <- plogis(gamma0 + gamma_y * df.obs$y + gamma_g*df.obs$g)
-        p.s.obs[df.obs$w_pred==0] <- 1 - p.s.obs[df.obs$w_pred==0]
+        yobs0 <- df.obs$y==0 
+        yobs1 <- df.obs$y==1
+        p.y.obs <- vector(mode='numeric', length=nrow(df.obs))
+        
+        p.y.obs[yobs1] <- plogis(B0 + Bxy * df.obs[yobs1]$x + Bzy*df.obs[yobs1]$z,log=T)
+        p.y.obs[yobs0] <- plogis(B0 + Bxy * df.obs[yobs0]$x + Bzy*df.obs[yobs0]$z,lower.tail=FALSE,log=T)
+
+        wobs0 <- df.obs$w_pred==0
+        wobs1 <- df.obs$w_pred==1
+        p.w.obs <- vector(mode='numeric', length=nrow(df.obs))
+
+        p.w.obs[wobs1] <- plogis(gamma0 + gamma_y * df.obs[wobs1]$y + gamma_z*df.obs[wobs1]$z + df.obs[wobs1]$z*df.obs[wobs1]$y* gamma_yz, log=T)
+        p.w.obs[wobs0] <- plogis(gamma0 + gamma_y * df.obs[wobs0]$y + gamma_z*df.obs[wobs0]$z + df.obs[wobs0]$z*df.obs[wobs0]$y* gamma_yz, lower.tail=FALSE, log=T)

         
         

-        p.obs <- p.s.obs * p.y.obs
+        p.obs <- p.w.obs + p.y.obs

 
         df.unobs <- df[is.na(y.obs)]
 
 
         df.unobs <- df[is.na(y.obs)]
 

-        p.unobs.1 <- plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g)*plogis(gamma0 + gamma_y + gamma_g*df.unobs$g)
-        p.unobs.0 <- (1-plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g))*plogis(gamma0 + gamma_g*df.unobs$g)
-        p.unobs <- p.unobs.1 + p.unobs.0
-        p.unobs[df.unobs$w_pred==0] <- 1 - p.unobs[df.unobs$w_pred==0]
+        p.unobs.0 <- vector(mode='numeric',length=nrow(df.unobs))
+        p.unobs.1 <- vector(mode='numeric',length=nrow(df.unobs))
+
+        wunobs.0 <- df.unobs$w_pred == 0
+        wunobs.1 <- df.unobs$w_pred == 1
+        
+        p.unobs.0[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.1]$z + df.unobs[wunobs.1]$z*gamma_yz, log=T)
+
+        p.unobs.0[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.0]$z + df.unobs[wunobs.0]$z*gamma_yz, lower.tail=FALSE, log=T)
+
+        p.unobs.1[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.1]$z, log=T)
+
+        p.unobs.1[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.0]$z, lower.tail=FALSE, log=T)
+
+        p.unobs <- colLogSumExps(rbind(p.unobs.1, p.unobs.0))

 
         p <- c(p.obs, p.unobs)
 
 
         p <- c(p.obs, p.unobs)
 

-        return(-1*(sum(log(p))))
+        return(-1*(sum(p)))

     }
 
     }
 

-    mlefit <- stats4::mle(minuslogl = nll, start = list(B0=1, Bxy=0,Bgy=0, gamma0=5, gamma_y=0, gamma_g=0))
+    mlefit <- mle2(minuslogl = nll, start = list(B0=0, Bxy=0,Bzy=0, gamma0=0, gamma_y=0, gamma_z=0, gamma_yz=0), control=list(maxit=1e6),method='L-BFGS-B')

 
     return(mlefit)
 }
 
 
     return(mlefit)
 }
 

-
-logistic <- function(x) {1/(1+exp(-1*x))}
-
-run_simulation_depvar <- function(df, result){
+run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y){

 
     accuracy <- df[,mean(w_pred==y)]
     result <- append(result, list(accuracy=accuracy))
 
 
     accuracy <- df[,mean(w_pred==y)]
     result <- append(result, list(accuracy=accuracy))
 

-    (model.true <- glm(y ~ x + g, data=df,family=binomial(link='logit')))
+    (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))

     true.ci.Bxy <- confint(model.true)['x',]
     true.ci.Bxy <- confint(model.true)['x',]

-    true.ci.Bgy <- confint(model.true)['g',]
+    true.ci.Bzy <- confint(model.true)['z',]

 
     result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
 
     result <- append(result, list(Bxy.est.true=coef(model.true)['x'],

-                                  Bgy.est.true=coef(model.true)['g'],
+                                  Bzy.est.true=coef(model.true)['z'],

                                   Bxy.ci.upper.true = true.ci.Bxy[2],
                                   Bxy.ci.lower.true = true.ci.Bxy[1],
                                   Bxy.ci.upper.true = true.ci.Bxy[2],
                                   Bxy.ci.lower.true = true.ci.Bxy[1],

-                                  Bgy.ci.upper.true = true.ci.Bgy[2],
-                                  Bgy.ci.lower.true = true.ci.Bgy[1]))
+                                  Bzy.ci.upper.true = true.ci.Bzy[2],
+                                  Bzy.ci.lower.true = true.ci.Bzy[1]))

                                   
                                   

-    (model.feasible <- glm(y.obs~x+g,data=df,family=binomial(link='logit')))
+    (model.feasible <- glm(y.obs~x+z,data=df,family=binomial(link='logit')))

 
     feasible.ci.Bxy <- confint(model.feasible)['x',]
     result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x'],
                                   Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
                                   Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
 
 
     feasible.ci.Bxy <- confint(model.feasible)['x',]
     result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x'],
                                   Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
                                   Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
 

-    feasible.ci.Bgy <- confint(model.feasible)['g',]
-    result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
-                                  Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
-                                  Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
+    feasible.ci.Bzy <- confint(model.feasible)['z',]
+    result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
+                                  Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
+                                  Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))

 
 

-    (model.naive <- glm(w_pred~x+g, data=df, family=binomial(link='logit')))
+    (model.naive <- glm(w_pred~x+z, data=df, family=binomial(link='logit')))

 
     naive.ci.Bxy <- confint(model.naive)['x',]
 
     naive.ci.Bxy <- confint(model.naive)['x',]

-    naive.ci.Bgy <- confint(model.naive)['g',]
+    naive.ci.Bzy <- confint(model.naive)['z',]

 
     result <- append(result, list(Bxy.est.naive=coef(model.naive)['x'],
 
     result <- append(result, list(Bxy.est.naive=coef(model.naive)['x'],

-                                  Bgy.est.naive=coef(model.naive)['g'],
+                                  Bzy.est.naive=coef(model.naive)['z'],

                                   Bxy.ci.upper.naive = naive.ci.Bxy[2],
                                   Bxy.ci.lower.naive = naive.ci.Bxy[1],
                                   Bxy.ci.upper.naive = naive.ci.Bxy[2],
                                   Bxy.ci.lower.naive = naive.ci.Bxy[1],

-                                  Bgy.ci.upper.naive = naive.ci.Bgy[2],
-                                  Bgy.ci.lower.naive = naive.ci.Bgy[1]))
+                                  Bzy.ci.upper.naive = naive.ci.Bzy[2],
+                                  Bzy.ci.lower.naive = naive.ci.Bzy[1]))

 
 
 
 

-    (model.naive.cont <- lm(w~x+g, data=df))
+    (model.naive.cont <- lm(w~x+z, data=df))

     naivecont.ci.Bxy <- confint(model.naive.cont)['x',]
     naivecont.ci.Bxy <- confint(model.naive.cont)['x',]

-    naivecont.ci.Bgy <- confint(model.naive.cont)['g',]
+    naivecont.ci.Bzy <- confint(model.naive.cont)['z',]

 
     ## my implementatoin of liklihood based correction
 
     ## my implementatoin of liklihood based correction

-    mod.caroll.lik <- logistic.correction.liklihood(df)
-    coef <- coef(mod.caroll.lik)
-    ci <- confint(mod.caroll.lik)

 
 

+    temp.df <- copy(df)
+    temp.df[,y:=y.obs]
+    mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+    fisher.info <- solve(mod.caroll.lik$hessian)
+    coef <- mod.caroll.lik$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96

     result <- append(result,
     result <- append(result,

-                     list(Bxy.est.mle = coef['Bxy'],
-                          Bxy.ci.upper.mle = ci['Bxy','97.5 %'],
-                          Bxy.ci.lower.mle = ci['Bxy','2.5 %'],
-                          Bgy.est.mle = coef['Bgy'],
-                          Bgy.ci.upper.mle = ci['Bgy','97.5 %'],
-                          Bgy.ci.lower.mle = ci['Bgy','2.5 %']))
-                          
+                     list(Bxy.est.mle = coef['x'],
+                          Bxy.ci.upper.mle = ci.upper['x'],
+                          Bxy.ci.lower.mle = ci.lower['x'],
+                          Bzy.est.mle = coef['z'],
+                          Bzy.ci.upper.mle = ci.upper['z'],
+                          Bzy.ci.lower.mle = ci.lower['z']))
+

 
     ## my implementatoin of liklihood based correction
 
     ## my implementatoin of liklihood based correction

-    mod.caroll.pseudo <- logistic.correction.pseudo(df)
-    coef <- coef(mod.caroll.pseudo)
-    ci <- confint(mod.caroll.pseudo)
+    mod.zhang <- zhang.mle.dv(df)
+    coef <- coef(mod.zhang)
+    ci <- confint(mod.zhang,method='quad')

 
     result <- append(result,
 
     result <- append(result,

-                     list(Bxy.est.pseudo = coef['Bxy'],
-                          Bxy.ci.upper.pseudo = ci['Bxy','97.5 %'],
-                          Bxy.ci.lower.pseudo = ci['Bxy','2.5 %'],
-                          Bgy.est.pseudo = coef['Bgy'],
-                          Bgy.ci.upper.pseudo = ci['Bgy','97.5 %'],
-                          Bgy.ci.lower.pseudo = ci['Bgy','2.5 %']))
+                     list(Bxy.est.zhang = coef['Bxy'],
+                          Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                          Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                          Bzy.est.zhang = coef['Bzy'],
+                          Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                          Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))

                           
 
     # amelia says use normal distribution for binary variables.
                           
 
     # amelia says use normal distribution for binary variables.

-    amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w_pred'))
-    mod.amelia.k <- zelig(y.obs~x+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
-    (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+    tryCatch({
+        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'))
+        mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+        (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+        est.x.mi <- coefse['x','Estimate']
+        est.x.se <- coefse['x','Std.Error']
+        result <- append(result,
+                         list(Bxy.est.amelia.full = est.x.mi,
+                              Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
+                              Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
+                              ))
+
+        est.z.mi <- coefse['z','Estimate']
+        est.z.se <- coefse['z','Std.Error']
+
+        result <- append(result,
+                         list(Bzy.est.amelia.full = est.z.mi,
+                              Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                              Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+                              ))
+
+    },
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error,'\n', e)
+    })

 
 

-    est.x.mi <- coefse['x','Estimate']
-    est.x.se <- coefse['x','Std.Error']
-    result <- append(result,
-                     list(Bxy.est.amelia.full = est.x.mi,
-                          Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
-                          Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
-                          ))
-
-    est.g.mi <- coefse['g','Estimate']
-    est.g.se <- coefse['g','Std.Error']
-
-    result <- append(result,
-                     list(Bgy.est.amelia.full = est.g.mi,
-                          Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
-                          Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
-                          ))

 
     return(result)
 
 }
 
 
     return(result)
 
 }
 

-run_simulation <-  function(df, result){
+
+## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle 
+run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){

 
     accuracy <- df[,mean(w_pred==x)]
     result <- append(result, list(accuracy=accuracy))
 
 
     accuracy <- df[,mean(w_pred==x)]
     result <- append(result, list(accuracy=accuracy))
 

-    (model.true <- lm(y ~ x + g, data=df))
+    (model.true <- lm(y ~ x + z, data=df))

     true.ci.Bxy <- confint(model.true)['x',]
     true.ci.Bxy <- confint(model.true)['x',]

-    true.ci.Bgy <- confint(model.true)['g',]
+    true.ci.Bzy <- confint(model.true)['z',]

 
     result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
 
     result <- append(result, list(Bxy.est.true=coef(model.true)['x'],

-                                  Bgy.est.true=coef(model.true)['g'],
+                                  Bzy.est.true=coef(model.true)['z'],

                                   Bxy.ci.upper.true = true.ci.Bxy[2],
                                   Bxy.ci.lower.true = true.ci.Bxy[1],
                                   Bxy.ci.upper.true = true.ci.Bxy[2],
                                   Bxy.ci.lower.true = true.ci.Bxy[1],

-                                  Bgy.ci.upper.true = true.ci.Bgy[2],
-                                  Bgy.ci.lower.true = true.ci.Bgy[1]))
+                                  Bzy.ci.upper.true = true.ci.Bzy[2],
+                                  Bzy.ci.lower.true = true.ci.Bzy[1]))

                                   
                                   

-    (model.feasible <- lm(y~x.obs+g,data=df))
+    (model.feasible <- lm(y~x.obs+z,data=df))

 
     feasible.ci.Bxy <- confint(model.feasible)['x.obs',]
     result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x.obs'],
                                   Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
                                   Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
 
 
     feasible.ci.Bxy <- confint(model.feasible)['x.obs',]
     result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x.obs'],
                                   Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
                                   Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
 

-    feasible.ci.Bgy <- confint(model.feasible)['g',]
-    result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
-                                  Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
-                                  Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
+    feasible.ci.Bzy <- confint(model.feasible)['z',]
+    result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
+                                  Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
+                                  Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))

 
 

-    (model.naive <- lm(y~w+g, data=df))
+    (model.naive <- lm(y~w_pred+z, data=df))

     
     

-    naive.ci.Bxy <- confint(model.naive)['w',]
-    naive.ci.Bgy <- confint(model.naive)['g',]
+    naive.ci.Bxy <- confint(model.naive)['w_pred',]
+    naive.ci.Bzy <- confint(model.naive)['z',]

 
 

-    result <- append(result, list(Bxy.est.naive=coef(model.naive)['w'],
-                                  Bgy.est.naive=coef(model.naive)['g'],
+    result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'],
+                                  Bzy.est.naive=coef(model.naive)['z'],

                                   Bxy.ci.upper.naive = naive.ci.Bxy[2],
                                   Bxy.ci.lower.naive = naive.ci.Bxy[1],
                                   Bxy.ci.upper.naive = naive.ci.Bxy[2],
                                   Bxy.ci.lower.naive = naive.ci.Bxy[1],

-                                  Bgy.ci.upper.naive = naive.ci.Bgy[2],
-                                  Bgy.ci.lower.naive = naive.ci.Bgy[1]))
+                                  Bzy.ci.upper.naive = naive.ci.Bzy[2],
+                                  Bzy.ci.lower.naive = naive.ci.Bzy[1]))

                                   
 
                                   
 

+    tryCatch({

     amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))
     amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))

-    mod.amelia.k <- zelig(y~x.obs+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+    mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)

     (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
 
     est.x.mi <- coefse['x.obs','Estimate']
     (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
 
     est.x.mi <- coefse['x.obs','Estimate']


@@ -215,15 +332,65 @@ run_simulation <-  function(df, result){
                           Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
                           ))
 
                           Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
                           ))
 

-    est.g.mi <- coefse['g','Estimate']
-    est.g.se <- coefse['g','Std.Error']
+    est.z.mi <- coefse['z','Estimate']
+    est.z.se <- coefse['z','Std.Error']

 
     result <- append(result,
 
     result <- append(result,

-                     list(Bgy.est.amelia.full = est.g.mi,
-                          Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
-                          Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
+                     list(Bzy.est.amelia.full = est.z.mi,
+                          Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                          Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se

                           ))
 
                           ))
 

+    },
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <-paste0(result$error,'\n', e)
+    }
+    )
+
+    tryCatch({
+        temp.df <- copy(df)
+        temp.df <- temp.df[,x:=x.obs]
+        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+        fisher.info <- solve(mod.caroll.lik$hessian)
+        coef <- mod.caroll.lik$par
+        ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+        ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+        
+        
+        result <- append(result,
+                         list(Bxy.est.mle = coef['x'],
+                              Bxy.ci.upper.mle = ci.upper['x'],
+                              Bxy.ci.lower.mle = ci.lower['x'],
+                              Bzy.est.mle = coef['z'],
+                              Bzy.ci.upper.mle = ci.upper['z'],
+                              Bzy.ci.lower.mle = ci.lower['z']))
+    },
+
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error,'\n', e)
+    })
+
+    tryCatch({
+
+        mod.zhang.lik <- zhang.mle.iv(df)
+        coef <- coef(mod.zhang.lik)
+        ci <- confint(mod.zhang.lik,method='quad')
+        result <- append(result,
+                         list(Bxy.est.zhang = coef['Bxy'],
+                              Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                              Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                              Bzy.est.zhang = coef['Bzy'],
+                              Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                              Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
+    },
+
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error,'\n', e)
+    })
+

     ## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
     ## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)
     ## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE)
     ## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
     ## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)
     ## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE)


@@ -255,10 +422,10 @@ run_simulation <-  function(df, result){
     df <- df[order(x.obs)]
     y <- df[,y]
     x <- df[,x.obs]
     df <- df[order(x.obs)]
     y <- df[,y]
     x <- df[,x.obs]

-    g <- df[,g]
-    w <- df[,w]
+    z <- df[,z]
+    w <- df[,w_pred]

     # gmm gets pretty close
     # gmm gets pretty close

-    (gmm.res <- predicted_covariates(y, x, g, w, v, train, p, max_iter=100, verbose=TRUE))
+    (gmm.res <- predicted_covariates(y, x, z, w, v, train, p, max_iter=100, verbose=TRUE))

 
     result <- append(result,
                      list(Bxy.est.gmm = gmm.res$beta[1,1],
 
     result <- append(result,
                      list(Bxy.est.gmm = gmm.res$beta[1,1],


@@ -268,28 +435,34 @@ run_simulation <-  function(df, result){
                           ))
 
     result <- append(result,
                           ))
 
     result <- append(result,

-                     list(Bgy.est.gmm = gmm.res$beta[2,1],
-                          Bgy.ci.upper.gmm = gmm.res$confint[2,2],
-                          Bgy.ci.lower.gmm = gmm.res$confint[2,1]))
+                     list(Bzy.est.gmm = gmm.res$beta[2,1],
+                          Bzy.ci.upper.gmm = gmm.res$confint[2,2],
+                          Bzy.ci.lower.gmm = gmm.res$confint[2,1]))

 
 
 
 

-    mod.calibrated.mle <- mecor(y ~ MeasError(w, reference = x.obs) + g, df, B=400, method='efficient')
+    tryCatch({
+    mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient')

     (mod.calibrated.mle)
     (mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
     result <- append(result, list(
                                  Bxy.est.mecor = mecor.ci['Estimate'],
     (mod.calibrated.mle)
     (mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
     result <- append(result, list(
                                  Bxy.est.mecor = mecor.ci['Estimate'],

-                                 Bxy.upper.mecor = mecor.ci['UCI'],
-                                 Bxy.lower.mecor = mecor.ci['LCI'])
+                                 Bxy.ci.upper.mecor = mecor.ci['UCI'],
+                                 Bxy.ci.lower.mecor = mecor.ci['LCI'])

                      )
 
                      )
 

-    (mecor.ci <- summary(mod.calibrated.mle)$c$ci['g',])
+    (mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',])

 
     result <- append(result, list(
 
     result <- append(result, list(

-                                 Bgy.est.mecor = mecor.ci['Estimate'],
-                                 Bgy.upper.mecor = mecor.ci['UCI'],
-                                 Bgy.lower.mecor = mecor.ci['LCI'])
+                                 Bzy.est.mecor = mecor.ci['Estimate'],
+                                 Bzy.ci.upper.mecor = mecor.ci['UCI'],
+                                 Bzy.ci.lower.mecor = mecor.ci['LCI'])

                      )
                      )

-
+    },
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error, '\n', e)
+    }
+    )

 ##    clean up memory
 ##    rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))
     
 ##    clean up memory
 ##    rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))