## print(mean(df[z==1]$x == df[z==1]$w_pred))
## print(mean(df$w_pred == df$x))
- odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(scale(df[x==1]$y)))
- odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(scale(df[x==0]$y)))
+ resids <- resid(lm(y~x + z))
+ odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(resids[x==1]))
+ odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(resids[x==0]))
## acc.x0 <- p.correct[df[,x==0]]
## acc.x1 <- p.correct[df[,x==1]]
df[,w_pred := as.integer(w > 0.5)]
- print(mean(df[z==0]$x == df[z==0]$w_pred))
- print(mean(df[z==1]$x == df[z==1]$w_pred))
+
print(mean(df$w_pred == df$x))
print(mean(df[y>=0]$w_pred == df[y>=0]$x))
print(mean(df[y<=0]$w_pred == df[y<=0]$x))
}
parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
-aparser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
parser <- add_argument(parser, "--seed", default=51, help='seed for the rng')
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x")
-parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.75)
+parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-1)
parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
args <- parse_args(parser)
## one way to do it is by adding correlation to x.obs and y that isn't in w.
## in other words, the model is missing an important feature of x.obs that's related to y.
-simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73){
+simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, log.likelihood.gain = 0.1){
+ set.seed(seed)
set.seed(seed)
# make w and y dependent
ystar <- Bzy * z + Bxy * x + B0
y <- rbinom(N,1,plogis(ystar))
- # glm(y ~ x + z, family="binomial")
-
df <- data.table(x=x,y=y,ystar=ystar,z=z)
if(m < N){
}
parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--N", default=10000, help="number of observations of w")
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
-parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72)
## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
-parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
-parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.01)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.01)
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y")
--- /dev/null
+### EXAMPLE 2_b: demonstrates how measurement error can lead to a type
+### sign error in a covariate This is the same as example 2, only
+### instead of x->k we have k->x. Even when you have a good
+### predictor, if it's biased against a covariate you can get the
+### wrong sign. Even when you include the proxy variable in the
+### regression. But with some ground truth and multiple imputation,
+### you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+
+library(predictionError)
+options(amelia.parallel="no", amelia.ncpus=1)
+
+source("irr_simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+####
+
+simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, coder_accuracy=0.9, seed=1){
+ set.seed(seed)
+ z <- rbinom(N, 1, 0.5)
+ # x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
+ xprime <- Bzx * z #+ x.var.epsilon
+ x <- rbinom(N,1,plogis(xprime))
+
+ y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)
+ y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
+ y <- Bzy * z + Bxy * x + y.epsilon
+
+ df <- data.table(x=x,y=y,z=z)
+
+ if(m < N){
+ df <- df[sample(nrow(df), m), x.obs := x]
+ } else {
+ df <- df[, x.obs := x]
+ }
+
+ df[ (!is.na(x.obs)) ,x.obs.0 := abs(x.obs - rbinom(.N, 1, 1-coder_accuracy))]
+ df[ (!is.na(x.obs)) ,x.obs.1 := abs(x.obs - rbinom(.N, 1, 1-coder_accuracy))]
+
+
+ ## how can you make a model with a specific accuracy?
+ w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy
+
+ ## how can you make a model with a specific accuracy, with a continuous latent variable.
+ # now it makes the same amount of mistake to each point, probably
+ # add mean0 noise to the odds.
+
+ w.noisey.odds = rlogis(N,qlogis(w0))
+ df[,w := plogis(w.noisey.odds)]
+ df[,w_pred:=as.integer(w > 0.5)]
+ (mean(df$x==df$w_pred))
+ return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=57, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
+# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
+parser <- add_argument(parser, "--coder_accuracy", help='how accurate is the predictive model?', default=0.8)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~x")
+
+# parser <- add_argument(parser, "--rater_formula", help='formula for the true variable', default="x.obs~x")
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=-0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
+
+args <- parse_args(parser)
+B0 <- 0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if (args$m < args$N){
+
+ df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance, prediction_accuracy=args$prediction_accuracy, coder_accuracy=args$coder_accuracy)
+
+ result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'outcome_formula'=args$outcome_formula, 'truth_formula'=args$truth_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, 'coder_accuracy'=args$coder_accuracy, error='')
+
+ outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula))
+
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+ if(file.exists(args$outfile)){
+ logdata <- read_feather(args$outfile)
+ logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+ } else {
+ logdata <- as.data.table(outline)
+ }
+
+ print(outline)
+ write_feather(logdata, args$outfile)
+ unlock(outfile_lock)
+}
--- /dev/null
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+ amelia.ncpus=1)
+setDTthreads(40)
+
+source("irr_dv_simulation_base.R")
+
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, coder_accuracy=0.8){
+ set.seed(seed)
+
+ # make w and y dependent
+ z <- rbinom(N, 1, 0.5)
+ x <- rbinom(N, 1, 0.5)
+
+ ystar <- Bzy * z + Bxy * x + B0
+ y <- rbinom(N,1,plogis(ystar))
+
+ # glm(y ~ x + z, family="binomial")
+
+ df <- data.table(x=x,y=y,ystar=ystar,z=z)
+
+ if(m < N){
+ df <- df[sample(nrow(df), m), y.obs := y]
+ } else {
+ df <- df[, y.obs := y]
+ }
+
+ df[ (!is.na(y.obs)) ,y.obs.0 := abs(y.obs - rbinom(.N, 1, 1-coder_accuracy))]
+ df[ (!is.na(y.obs)) ,y.obs.1 := abs(y.obs - rbinom(.N, 1, 1-coder_accuracy))]
+
+ odds.y1 <- qlogis(prediction_accuracy)
+ odds.y0 <- qlogis(prediction_accuracy,lower.tail=F)
+
+ df[y==0,w:=plogis(rlogis(.N,odds.y0))]
+ df[y==1,w:=plogis(rlogis(.N,odds.y1))]
+
+ df[,w_pred := as.integer(w > 0.5)]
+
+ print(mean(df[x==0]$y == df[x==0]$w_pred))
+ print(mean(df[x==1]$y == df[x==1]$w_pred))
+ print(mean(df$w_pred == df$y))
+ return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72)
+## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
+## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y")
+parser <- add_argument(parser, "--coder_accuracy", help='How accurate are the coders?', default=0.8)
+
+args <- parse_args(parser)
+
+B0 <- 0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+
+
+if(args$m < args$N){
+ df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$coder_accuracy)
+
+# result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+ result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+
+ outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
+
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+
+ if(file.exists(args$outfile)){
+ logdata <- read_feather(args$outfile)
+ logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+ } else {
+ logdata <- as.data.table(outline)
+ }
+
+ print(outline)
+ write_feather(logdata, args$outfile)
+ unlock(outfile_lock)
+}
SHELL=bash
-Ns=[1000, 2000, 4000, 8000]
+Ns=[1000, 2000, 4000]
ms=[100, 200, 400, 800]
-seeds=[$(shell seq -s, 1 100)]
+seeds=[$(shell seq -s, 1 250)]
explained_variances=[0.1]
-all:remembr.RDS
+all:remembr.RDS remember_irr.RDS
+supplement: remember_robustness_misspec.RDS
-srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1
+srun=sbatch --wait --verbose run_job.sbatch
joblists:example_1_jobs example_2_jobs example_3_jobs
# test_true_z_jobs: test_true_z.R simulation_base.R
-# grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
+# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
# test_true_z.feather: test_true_z_jobs
# rm -f test_true_z.feather
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs
-example_1_jobs: 01_two_covariates.R simulation_base.R
- grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs
+example_1_jobs: 01_two_covariates.R simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.3]}' --outfile example_1_jobs
example_1.feather: example_1_jobs
rm -f example_1.feather
sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
-example_2_jobs: 02_indep_differential.R simulation_base.R
- grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y*z*x"], "truth_formula":["x~z"]}' --outfile example_2_jobs
+example_2_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y*z*x"]}' --outfile example_2_jobs
example_2.feather: example_2_jobs
rm -f example_2.feather
- sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
+ sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
# example_2_B_jobs: example_2_B.R
-# grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
+# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
# example_2_B.feather: example_2_B_jobs
# rm -f example_2_B.feather
# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
-example_3_jobs: 03_depvar.R simulation_base.R
- grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
+example_3_jobs: 03_depvar.R simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "Bxy":[0.01],"Bzy":[-0.01],"seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
example_3.feather: example_3_jobs
rm -f example_3.feather
sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
-example_4_jobs: 04_depvar_differential.R simulation_base.R
- grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "y_explained_variance":${explained_variances}}' --outfile example_4_jobs
+example_4_jobs: 04_depvar_differential.R simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"Bxy":[0.01],"Bzy":[-0.01],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "y_explained_variance":${explained_variances}}' --outfile example_4_jobs
example_4.feather: example_4_jobs
rm -f example_4.feather
sbatch --wait --verbose --array=1-$(shell cat example_4_jobs | wc -l) run_simulation.sbatch 0 example_4_jobs
-remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R
+
+remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R summarize_estimator.R
rm -f remembr.RDS
${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
${srun} Rscript plot_dv_example.R --infile example_4.feather --name "plot.df.example.4"
+
+irr_Ns = ${Ns}
+irr_ms = ${ms}
+irr_seeds=${seeds}
+irr_explained_variances=${explained_variances}
+
+example_5_jobs: 05_irr_indep.R irr_simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 05_irr_indep.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_5.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_5_jobs
+
+example_5.feather:example_5_jobs
+ rm -f example_5.feather
+ sbatch --wait --verbose --array=1-$(shell cat example_5_jobs | wc -l) run_simulation.sbatch 0 example_5_jobs
+
+
+example_6_jobs: 06_irr_dv.R irr_dv_simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 06_irr_dv.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_6.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_6_jobs
+
+example_6.feather:example_6_jobs
+ rm -f example_6.feather
+ sbatch --wait --verbose --array=1-$(shell cat example_6_jobs | wc -l) run_simulation.sbatch 0 example_6_jobs
+
+remember_irr.RDS: example_5.feather example_6.feather plot_irr_example.R plot_irr_dv_example.R summarize_estimator.R
+ rm -f remember_irr.RDS
+ sbatch --wait --verbose run_job.sbatch Rscript plot_irr_example.R --infile example_5.feather --name "plot.df.example.5"
+ sbatch --wait --verbose run_job.sbatch Rscript plot_irr_dv_example.R --infile example_6.feather --name "plot.df.example.6"
+
+
+
+robustness_1_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py
+ sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs
+
+
+
+robustness_1.feather: robustness_1_jobs
+ rm -f robustness_1.feather
+ sbatch --wait --verbose --array=1-$(shell cat robustness_1_jobs | wc -l) run_simulation.sbatch 0 robustness_1_jobs
+
+robustness_1_dv_jobs: simulation_base.R 04_depvar_differential.R grid_sweep.py
+ ${srun} bash -c "source ~/.bashrc && grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict \"{'N':${Ns},'m':${ms}, 'seed':${seeds}, 'outfile':['robustness_1_dv.feather'], 'y_explained_variance':${explained_variances}, 'proxy_formula':['w_pred~y']}\" --outfile robustness_1_dv_jobs"
+
+
+robustness_1_dv.feather: robustness_1_dv_jobs
+ rm -f robustness_1_dv.feather
+ sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 robustness_1_dv_jobs
+
+
+remember_robustness_misspec.RDS: robustness_1.feather robustness_1_dv.feather
+ rm -f remember_robustness_misspec.RDS
+ sbatch --wait --verbose run_job.sbatch Rscript plot_example.R --infile robustness_1.feather --name "plot.df.robustness.1" --remember-file "remember_robustness_misspec.RDS"
+ sbatch --wait --verbose run_job.sbatch Rscript plot_dv_example.R --infile robustness_1_dv.feather --name "plot.df.robustness.1.dv" --remember-file "remember_robustness_mispec.RDS"
+
+
clean:
rm *.feather
rm -f remembr.RDS
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
# example_2_B_mecor_jobs:
-# grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs
+# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs
# example_2_B_mecor.feather:example_2_B_mecor.R example_2_B_mecor_jobs
# rm -f example_2_B_mecor.feather
+.PHONY: supplement
import fire
from itertools import product
+import pyRemembeR
-def main(command, arg_dict, outfile):
+def main(command, arg_dict, outfile, remember_file='remember_grid_sweep.RDS'):
+ remember = pyRemembeR.remember.Remember()
+ remember.set_file(remember_file)
+ remember[outfile] = arg_dict
+ remember.save_to_r()
keys = []
values = []
--- /dev/null
+library(matrixStats) # for numerically stable logsumexps
+
+options(amelia.parallel="no",
+ amelia.ncpus=1)
+library(Amelia)
+
+source("measerr_methods.R") ## for my more generic function.
+
+run_simulation_depvar <- function(df, result, outcome_formula = y ~ x + z, rater_formula = y.obs ~ x, proxy_formula = w_pred ~ y){
+
+ accuracy <- df[,mean(w_pred==y)]
+ result <- append(result, list(accuracy=accuracy))
+
+ (model.true <- glm(y ~ x + z, data=df, family=binomial(link='logit')))
+ true.ci.Bxy <- confint(model.true)['x',]
+ true.ci.Bzy <- confint(model.true)['z',]
+
+ result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
+ Bzy.est.true=coef(model.true)['z'],
+ Bxy.ci.upper.true = true.ci.Bxy[2],
+ Bxy.ci.lower.true = true.ci.Bxy[1],
+ Bzy.ci.upper.true = true.ci.Bzy[2],
+ Bzy.ci.lower.true = true.ci.Bzy[1]))
+
+
+
+ loa0.feasible <- glm(y.obs.0 ~ x + z, data = df[!(is.na(y.obs.0))], family=binomial(link='logit'))
+
+ loa0.ci.Bxy <- confint(loa0.feasible)['x',]
+ loa0.ci.Bzy <- confint(loa0.feasible)['z',]
+
+ result <- append(result, list(Bxy.est.loa0.feasible=coef(loa0.feasible)['x'],
+ Bzy.est.loa0.feasible=coef(loa0.feasible)['z'],
+ Bxy.ci.upper.loa0.feasible = loa0.ci.Bxy[2],
+ Bxy.ci.lower.loa0.feasible = loa0.ci.Bxy[1],
+ Bzy.ci.upper.loa0.feasible = loa0.ci.Bzy[2],
+ Bzy.ci.lower.loa0.feasible = loa0.ci.Bzy[1]))
+
+
+ df.loa0.mle <- copy(df)
+ df.loa0.mle[,y:=y.obs.0]
+ loa0.mle <- measerr_mle_dv(df.loa0.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+ fisher.info <- solve(loa0.mle$hessian)
+ coef <- loa0.mle$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ result <- append(result, list(Bxy.est.loa0.mle=coef['x'],
+ Bzy.est.loa0.mle=coef['z'],
+ Bxy.ci.upper.loa0.mle = ci.upper['x'],
+ Bxy.ci.lower.loa0.mle = ci.lower['x'],
+ Bzy.ci.upper.loa0.mle = ci.upper['z'],
+ Bzy.ci.lower.loa0.mle = ci.upper['z']))
+
+ loco.feasible <- glm(y.obs.0 ~ x + z, data = df[(!is.na(y.obs.0)) & (y.obs.1 == y.obs.0)], family=binomial(link='logit'))
+
+ loco.feasible.ci.Bxy <- confint(loco.feasible)['x',]
+ loco.feasible.ci.Bzy <- confint(loco.feasible)['z',]
+
+ result <- append(result, list(Bxy.est.loco.feasible=coef(loco.feasible)['x'],
+ Bzy.est.loco.feasible=coef(loco.feasible)['z'],
+ Bxy.ci.upper.loco.feasible = loco.feasible.ci.Bxy[2],
+ Bxy.ci.lower.loco.feasible = loco.feasible.ci.Bxy[1],
+ Bzy.ci.upper.loco.feasible = loco.feasible.ci.Bzy[2],
+ Bzy.ci.lower.loco.feasible = loco.feasible.ci.Bzy[1]))
+
+
+ df.loco.mle <- copy(df)
+ df.loco.mle[,y.obs:=NA]
+ df.loco.mle[(y.obs.0)==(y.obs.1),y.obs:=y.obs.0]
+ df.loco.mle[,y.true:=y]
+ df.loco.mle[,y:=y.obs]
+ print(df.loco.mle[!is.na(y.obs.1),mean(y.true==y,na.rm=TRUE)])
+ loco.mle <- measerr_mle_dv(df.loco.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+ fisher.info <- solve(loco.mle$hessian)
+ coef <- loco.mle$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ result <- append(result, list(Bxy.est.loco.mle=coef['x'],
+ Bzy.est.loco.mle=coef['z'],
+ Bxy.ci.upper.loco.mle = ci.upper['x'],
+ Bxy.ci.lower.loco.mle = ci.lower['x'],
+ Bzy.ci.upper.loco.mle = ci.upper['z'],
+ Bzy.ci.lower.loco.mle = ci.lower['z']))
+
+ print(rater_formula)
+ print(proxy_formula)
+
+ ## mle.irr <- measerr_irr_mle( df, outcome_formula = outcome_formula, rater_formula = rater_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+
+ ## fisher.info <- solve(mle.irr$hessian)
+ ## coef <- mle.irr$par
+ ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ ## result <- append(result,
+ ## list(Bxy.est.mle = coef['x'],
+ ## Bxy.ci.upper.mle = ci.upper['x'],
+ ## Bxy.ci.lower.mle = ci.lower['x'],
+ ## Bzy.est.mle = coef['z'],
+ ## Bzy.ci.upper.mle = ci.upper['z'],
+ ## Bzy.ci.lower.mle = ci.lower['z']))
+
+ return(result)
+
+}
--- /dev/null
+library(matrixStats) # for numerically stable logsumexps
+
+options(amelia.parallel="no",
+ amelia.ncpus=1)
+library(Amelia)
+
+source("measerr_methods.R") ## for my more generic function.
+
+run_simulation <- function(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ x, truth_formula = x ~ z){
+
+ accuracy <- df[,mean(w_pred==x)]
+ result <- append(result, list(accuracy=accuracy))
+
+ (model.true <- lm(y ~ x + z, data=df))
+ true.ci.Bxy <- confint(model.true)['x',]
+ true.ci.Bzy <- confint(model.true)['z',]
+
+ result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
+ Bzy.est.true=coef(model.true)['z'],
+ Bxy.ci.upper.true = true.ci.Bxy[2],
+ Bxy.ci.lower.true = true.ci.Bxy[1],
+ Bzy.ci.upper.true = true.ci.Bzy[2],
+ Bzy.ci.lower.true = true.ci.Bzy[1]))
+
+
+
+ loa0.feasible <- lm(y ~ x.obs.0 + z, data = df[!(is.na(x.obs.1))])
+
+ loa0.ci.Bxy <- confint(loa0.feasible)['x.obs.0',]
+ loa0.ci.Bzy <- confint(loa0.feasible)['z',]
+
+ result <- append(result, list(Bxy.est.loa0.feasible=coef(loa0.feasible)['x.obs.0'],
+ Bzy.est.loa0.feasible=coef(loa0.feasible)['z'],
+ Bxy.ci.upper.loa0.feasible = loa0.ci.Bxy[2],
+ Bxy.ci.lower.loa0.feasible = loa0.ci.Bxy[1],
+ Bzy.ci.upper.loa0.feasible = loa0.ci.Bzy[2],
+ Bzy.ci.lower.loa0.feasible = loa0.ci.Bzy[1]))
+
+
+ df.loa0.mle <- copy(df)
+ df.loa0.mle[,x:=x.obs.0]
+ loa0.mle <- measerr_mle(df.loa0.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+ fisher.info <- solve(loa0.mle$hessian)
+ coef <- loa0.mle$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ result <- append(result, list(Bxy.est.loa0.mle=coef['x'],
+ Bzy.est.loa0.mle=coef['z'],
+ Bxy.ci.upper.loa0.mle = ci.upper['x'],
+ Bxy.ci.lower.loa0.mle = ci.lower['x'],
+ Bzy.ci.upper.loa0.mle = ci.upper['z'],
+ Bzy.ci.lower.loa0.mle = ci.upper['z']))
+
+ loco.feasible <- lm(y ~ x.obs.1 + z, data = df[(!is.na(x.obs.1)) & (x.obs.1 == x.obs.0)])
+
+ loco.feasible.ci.Bxy <- confint(loco.feasible)['x.obs.1',]
+ loco.feasible.ci.Bzy <- confint(loco.feasible)['z',]
+
+ result <- append(result, list(Bxy.est.loco.feasible=coef(loco.feasible)['x.obs.1'],
+ Bzy.est.loco.feasible=coef(loco.feasible)['z'],
+ Bxy.ci.upper.loco.feasible = loco.feasible.ci.Bxy[2],
+ Bxy.ci.lower.loco.feasible = loco.feasible.ci.Bxy[1],
+ Bzy.ci.upper.loco.feasible = loco.feasible.ci.Bzy[2],
+ Bzy.ci.lower.loco.feasible = loco.feasible.ci.Bzy[1]))
+
+
+ df.loco.mle <- copy(df)
+ df.loco.mle[,x.obs:=NA]
+ df.loco.mle[(x.obs.0)==(x.obs.1),x.obs:=x.obs.0]
+ df.loco.mle[,x.true:=x]
+ df.loco.mle[,x:=x.obs]
+ print(df.loco.mle[!is.na(x.obs.1),mean(x.true==x,na.rm=TRUE)])
+ loco.mle <- measerr_mle(df.loco.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+ fisher.info <- solve(loco.mle$hessian)
+ coef <- loco.mle$par
+ ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ result <- append(result, list(Bxy.est.loco.mle=coef['x'],
+ Bzy.est.loco.mle=coef['z'],
+ Bxy.ci.upper.loco.mle = ci.upper['x'],
+ Bxy.ci.lower.loco.mle = ci.lower['x'],
+ Bzy.ci.upper.loco.mle = ci.upper['z'],
+ Bzy.ci.lower.loco.mle = ci.lower['z']))
+
+ ## print(rater_formula)
+ ## print(proxy_formula)
+ ## mle.irr <- measerr_irr_mle( df, outcome_formula = outcome_formula, rater_formula = rater_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+
+ ## fisher.info <- solve(mle.irr$hessian)
+ ## coef <- mle.irr$par
+ ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+ ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+ ## result <- append(result,
+ ## list(Bxy.est.mle = coef['x'],
+ ## Bxy.ci.upper.mle = ci.upper['x'],
+ ## Bxy.ci.lower.mle = ci.lower['x'],
+ ## Bzy.est.mle = coef['z'],
+ ## Bzy.ci.upper.mle = ci.upper['z'],
+ ## Bzy.ci.lower.mle = ci.lower['z']))
+
+ return(result)
+
+}
library(formula.tools)
library(matrixStats)
-
+library(bbmle)
## df: dataframe to model
## outcome_formula: formula for y | x, z
## outcome_family: family for y | x, z
## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
-measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
nll <- function(params){
df.obs <- model.frame(outcome_formula, df)
start <- rep(0.1,length(params))
names(start) <- params
- fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ if(method=='optim'){
+ fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ } else {
+ quoted.names <- gsub("[\\(\\)]",'',names(start))
+ print(quoted.names)
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+ measerr_mle_nll <- eval(parse(text=text))
+ names(start) <- quoted.names
+ names(lower) <- quoted.names
+ fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ }
return(fit)
}
-measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
+## Experimental, and not necessary if errors are independent.
+measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
+
+ ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
+
+ ## probability of y given observed data.
+ df.obs <- df[!is.na(x.obs.1)]
+ proxy.variable <- all.vars(proxy_formula)[1]
+ df.x.obs.1 <- copy(df.obs)[,x:=1]
+ df.x.obs.0 <- copy(df.obs)[,x:=0]
+ y.obs <- df.obs[,y]
+
+ nll <- function(params){
+ outcome.model.matrix.x.obs.0 <- model.matrix(outcome_formula, df.x.obs.0)
+ outcome.model.matrix.x.obs.1 <- model.matrix(outcome_formula, df.x.obs.1)
+
+ param.idx <- 1
+ n.outcome.model.covars <- dim(outcome.model.matrix.x.obs.0)[2]
+ outcome.params <- params[param.idx:n.outcome.model.covars]
+ param.idx <- param.idx + n.outcome.model.covars
+
+ sigma.y <- params[param.idx]
+ param.idx <- param.idx + 1
+
+ ll.y.x.obs.0 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.0),sd=sigma.y, log=TRUE)
+ ll.y.x.obs.1 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.1),sd=sigma.y, log=TRUE)
+
+ ## assume that the two coders are statistically independent conditional on x
+ ll.x.obs.0.x0 <- vector(mode='numeric', length=nrow(df.obs))
+ ll.x.obs.1.x0 <- vector(mode='numeric', length=nrow(df.obs))
+ ll.x.obs.0.x1 <- vector(mode='numeric', length=nrow(df.obs))
+ ll.x.obs.1.x1 <- vector(mode='numeric', length=nrow(df.obs))
+
+ rater.model.matrix.x.obs.0 <- model.matrix(rater_formula, df.x.obs.0)
+ rater.model.matrix.x.obs.1 <- model.matrix(rater_formula, df.x.obs.1)
+
+ n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
+ rater.0.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
+ param.idx <- param.idx + n.rater.model.covars
+
+ rater.1.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
+ param.idx <- param.idx + n.rater.model.covars
+
+ # probability of rater 0 if x is 0 or 1
+ ll.x.obs.0.x0[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
+ ll.x.obs.0.x0[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
+ ll.x.obs.0.x1[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==1,]), log=TRUE)
+ ll.x.obs.0.x1[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
+
+ # probability of rater 1 if x is 0 or 1
+ ll.x.obs.1.x0[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==1,]), log=TRUE)
+ ll.x.obs.1.x0[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
+ ll.x.obs.1.x1[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==1,]), log=TRUE)
+ ll.x.obs.1.x1[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
+
+ proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.x.obs.0)
+ proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.x.obs.1)
+
+ n.proxy.model.covars <- dim(proxy.model.matrix.x0)[2]
+ proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+ param.idx <- param.idx + n.proxy.model.covars
+
+ proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
+
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+ ll.w.obs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
+ ll.w.obs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
+
+ # proxy_formula likelihood using logistic regression
+ ll.w.obs.x0[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==1,]),log=TRUE)
+ ll.w.obs.x0[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+
+ ll.w.obs.x1[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==1,]),log=TRUE)
+ ll.w.obs.x1[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ ## assume that the probability of x is a logistic regression depending on z
+ truth.model.matrix.obs <- model.matrix(truth_formula, df.obs)
+ n.truth.params <- dim(truth.model.matrix.obs)[2]
+ truth.params <- params[param.idx:(n.truth.params + param.idx - 1)]
+
+ ll.obs.x0 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE)
+ ll.obs.x1 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE, lower.tail=FALSE)
+
+ ll.obs <- colLogSumExps(rbind(ll.y.x.obs.0 + ll.x.obs.0.x0 + ll.x.obs.1.x0 + ll.obs.x0 + ll.w.obs.x0,
+ ll.y.x.obs.1 + ll.x.obs.0.x1 + ll.x.obs.1.x1 + ll.obs.x1 + ll.w.obs.x1))
+
+ ### NOW FOR THE FUN PART. Likelihood of the unobserved data.
+ ### we have to integrate out x.obs.0, x.obs.1, and x.
+
+
+ ## THE OUTCOME
+ df.unobs <- df[is.na(x.obs)]
+ df.x.unobs.0 <- copy(df.unobs)[,x:=0]
+ df.x.unobs.1 <- copy(df.unobs)[,x:=1]
+ y.unobs <- df.unobs$y
+
+ outcome.model.matrix.x.unobs.0 <- model.matrix(outcome_formula, df.x.unobs.0)
+ outcome.model.matrix.x.unobs.1 <- model.matrix(outcome_formula, df.x.unobs.1)
+
+ ll.y.unobs.x0 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.0), sd=sigma.y, log=TRUE)
+ ll.y.unobs.x1 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.1), sd=sigma.y, log=TRUE)
+
+
+ ## THE UNLABELED DATA
- measrr_mle_nll <- function(params){
- df.obs <- model.frame(outcome_formula, df)
+ ## assume that the two coders are statistically independent conditional on x
+ ll.x.unobs.0.x0 <- vector(mode='numeric', length=nrow(df.unobs))
+ ll.x.unobs.1.x0 <- vector(mode='numeric', length=nrow(df.unobs))
+ ll.x.unobs.0.x1 <- vector(mode='numeric', length=nrow(df.unobs))
+ ll.x.unobs.1.x1 <- vector(mode='numeric', length=nrow(df.unobs))
+
+ df.x.unobs.0[,x.obs := 1]
+ df.x.unobs.1[,x.obs := 1]
+
+ rater.model.matrix.x.unobs.0 <- model.matrix(rater_formula, df.x.unobs.0)
+ rater.model.matrix.x.unobs.1 <- model.matrix(rater_formula, df.x.unobs.1)
+
+
+ ## # probability of rater 0 if x is 0 or 1
+ ## ll.x.unobs.0.x0 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
+ ## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+
+ ## ll.x.unobs.0.x1 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
+ ## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+
+ ## # probability of rater 1 if x is 0 or 1
+ ## ll.x.unobs.1.x0 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
+ ## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+
+ ## ll.x.unobs.1.x1 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
+ ## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+
+
+ proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
+ proxy.model.matrix.x0.unobs <- model.matrix(proxy_formula, df.x.unobs.0)
+ proxy.model.matrix.x1.unobs <- model.matrix(proxy_formula, df.x.unobs.1)
+
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+ ll.w.unobs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
+ ll.w.unobs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
+
+
+ # proxy_formula likelihood using logistic regression
+ ll.w.unobs.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==1,]),log=TRUE)
+ ll.w.unobs.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+
+ ll.w.unobs.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==1,]),log=TRUE)
+ ll.w.unobs.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+ }
+
+ truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs)
+
+ ll.unobs.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
+ ll.unobs.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
+
+ ll.unobs <- colLogSumExps(rbind(ll.unobs.x0 + ll.w.unobs.x0 + ll.y.unobs.x0,
+ ll.unobs.x1 + ll.w.unobs.x1 + ll.y.unobs.x1))
+
+ return(-1 *( sum(ll.obs) + sum(ll.unobs)))
+ }
+
+ outcome.params <- colnames(model.matrix(outcome_formula,df))
+ lower <- rep(-Inf, length(outcome.params))
+
+ if(outcome_family$family=='gaussian'){
+ params <- c(outcome.params, 'sigma_y')
+ lower <- c(lower, 0.00001)
+ } else {
+ params <- outcome.params
+ }
+
+ rater.0.params <- colnames(model.matrix(rater_formula,df))
+ params <- c(params, paste0('rater_0',rater.0.params))
+ lower <- c(lower, rep(-Inf, length(rater.0.params)))
+
+ rater.1.params <- colnames(model.matrix(rater_formula,df))
+ params <- c(params, paste0('rater_1',rater.1.params))
+ lower <- c(lower, rep(-Inf, length(rater.1.params)))
+
+ proxy.params <- colnames(model.matrix(proxy_formula, df))
+ params <- c(params, paste0('proxy_',proxy.params))
+ lower <- c(lower, rep(-Inf, length(proxy.params)))
+
+ truth.params <- colnames(model.matrix(truth_formula, df))
+ params <- c(params, paste0('truth_', truth.params))
+ lower <- c(lower, rep(-Inf, length(truth.params)))
+ start <- rep(0.1,length(params))
+ names(start) <- params
+
+
+ if(method=='optim'){
+ fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ } else {
+
+ quoted.names <- gsub("[\\(\\)]",'',names(start))
+ print(quoted.names)
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+ measerr_mle_nll <- eval(parse(text=text))
+ names(start) <- quoted.names
+ names(lower) <- quoted.names
+ fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ }
+
+ return(fit)
+}
+
+
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
+
+ measerr_mle_nll <- function(params){
+ df.obs <- model.frame(outcome_formula, df)
proxy.variable <- all.vars(proxy_formula)[1]
proxy.model.matrix <- model.matrix(proxy_formula, df)
-
response.var <- all.vars(outcome_formula)[1]
y.obs <- with(df.obs,eval(parse(text=response.var)))
-
+
outcome.model.matrix <- model.matrix(outcome_formula, df)
param.idx <- 1
sigma.y <- params[param.idx]
param.idx <- param.idx + 1
- # outcome_formula likelihood using linear regression
+ # outcome_formula likelihood using linear regression
ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
}
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
- # proxy_formula likelihood using logistic regression
+ # proxy_formula likelihood using logistic regression
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
}
if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
- # truth_formula likelihood using logistic regression
+ # truth_formula likelihood using logistic regression
ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
}
- # add the three likelihoods
+ # add the three likelihoods
ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
## likelihood for the predicted data
outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
if(outcome_family$family=="gaussian"){
- # likelihood of outcome
- ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
- ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+ # likelihood of outcome
+ ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+ ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
}
if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
- # likelihood of proxy
+ # likelihood of proxy
ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
if(truth_family$link=='logit'){
truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
- # likelihood of truth
+ # likelihood of truth
ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
}
lower <- c(lower, rep(-Inf, length(truth.params)))
start <- rep(0.1,length(params))
names(start) <- params
-
- fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+
+ if(method=='optim'){
+ fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ } else { # method='mle2'
+
+ quoted.names <- gsub("[\\(\\)]",'',names(start))
+
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
+
+ measerr_mle_nll_mle <- eval(parse(text=text))
+ names(start) <- quoted.names
+ names(lower) <- quoted.names
+ fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ }
return(fit)
}
+
parser <- arg_parser("Simulate data and fit corrected models.")
parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.")
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
args <- parse_args(parser)
-
-
-summarize.estimator <- function(df, suffix='naive', coefname='x'){
-
- part <- df[,c('N',
- 'm',
- 'Bxy',
- paste0('B',coefname,'y.est.',suffix),
- paste0('B',coefname,'y.ci.lower.',suffix),
- paste0('B',coefname,'y.ci.upper.',suffix),
- 'y_explained_variance',
- 'Bzy'
- ),
- with=FALSE]
+## summarize.estimator <- function(df, suffix='naive', coefname='x'){
+
+## part <- df[,c('N',
+## 'm',
+## 'Bxy',
+## paste0('B',coefname,'y.est.',suffix),
+## paste0('B',coefname,'y.ci.lower.',suffix),
+## paste0('B',coefname,'y.ci.upper.',suffix),
+## 'y_explained_variance',
+## 'Bzy'
+## ),
+## with=FALSE]
- true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
- zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
- bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
- sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
-
- part <- part[,':='(true.in.ci = true.in.ci,
- zero.in.ci = zero.in.ci,
- bias=bias,
- sign.correct =sign.correct)]
-
- part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
- var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
- est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
- est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable=coefname,
- method=suffix
- ),
- by=c("N","m",'Bzy','y_explained_variance')
- ]
+## true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+## zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+## bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+## sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+## part <- part[,':='(true.in.ci = true.in.ci,
+## zero.in.ci = zero.in.ci,
+## bias=bias,
+## sign.correct =sign.correct)]
+
+## part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+## mean.bias = mean(bias),
+## mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+## var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+## est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
+## est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
+## N.sims = .N,
+## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+## variable=coefname,
+## method=suffix
+## ),
+## by=c("N","m",'Bzy','y_explained_variance')
+## ]
- return(part.plot)
-}
+## return(part.plot)
+## }
+source("summarize_estimator.R")
build_plot_dataset <- function(df){
return(plot.df)
}
-
-df <- read_feather(args$infile)
-plot.df <- build_plot_dataset(df)
+change.remember.file(args$remember_file, clear=TRUE)
+sims.df <- read_feather(args$infile)
+sims.df[,Bzx:=NA]
+sims.df[,accuracy_imbalance_difference:=NA]
+plot.df <- build_plot_dataset(sims.df)
remember(plot.df,args$name)
+set.remember.prefix(gsub("plot.df.","",args$name))
+
+remember(median(sims.df$cor.xz),'med.cor.xz')
+remember(median(sims.df$accuracy),'med.accuracy')
+remember(median(sims.df$error.cor.x),'med.error.cor.x')
+remember(median(sims.df$lik.ratio),'med.lik.ratio')
+
+
+
## df[gmm.ER_pval<0.05]
## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
library(filelock)
library(argparser)
+source("summarize_estimator.R")
+
+
parser <- arg_parser("Simulate data and fit corrected models.")
parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.")
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
args <- parse_args(parser)
-summarize.estimator <- function(df, suffix='naive', coefname='x'){
-
- part <- df[,c('N',
- 'm',
- 'Bxy',
- paste0('B',coefname,'y.est.',suffix),
- paste0('B',coefname,'y.ci.lower.',suffix),
- paste0('B',coefname,'y.ci.upper.',suffix),
- 'y_explained_variance',
- 'Bzx',
- 'Bzy',
- 'accuracy_imbalance_difference'
- ),
- with=FALSE]
+
+
+## summarize.estimator <- function(df, suffix='naive', coefname='x'){
+
+## part <- df[,c('N',
+## 'm',
+## 'Bxy',
+## paste0('B',coefname,'y.est.',suffix),
+## paste0('B',coefname,'y.ci.lower.',suffix),
+## paste0('B',coefname,'y.ci.upper.',suffix),
+## 'y_explained_variance',
+## 'Bzx',
+## 'Bzy',
+## 'accuracy_imbalance_difference'
+## ),
+## with=FALSE]
- true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
- zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
- bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
- sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
-
- part <- part[,':='(true.in.ci = true.in.ci,
- zero.in.ci = zero.in.ci,
- bias=bias,
- sign.correct =sign.correct)]
-
- part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
- mean.bias = mean(bias),
- mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
- var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
- est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
- est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
- N.sims = .N,
- p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
- variable=coefname,
- method=suffix
- ),
- by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
- ]
+## true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+## zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+## bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+## sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+## part <- part[,':='(true.in.ci = true.in.ci,
+## zero.in.ci = zero.in.ci,
+## bias=bias,
+## sign.correct =sign.correct)]
+
+## part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+## mean.bias = mean(bias),
+## mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+## var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+## est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
+## est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
+## N.sims = .N,
+## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+## variable=coefname,
+## method=suffix
+## ),
+## by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
+## ]
- return(part.plot)
-}
+## return(part.plot)
+## }
build_plot_dataset <- function(df){
}
-plot.df <- read_feather(args$infile)
-print(unique(plot.df$N))
+sims.df <- read_feather(args$infile)
+print(unique(sims.df$N))
# df <- df[apply(df,1,function(x) !any(is.na(x)))]
-if(!('Bzx' %in% names(plot.df)))
- plot.df[,Bzx:=NA]
+if(!('Bzx' %in% names(sims.df)))
+ sims.df[,Bzx:=NA]
-if(!('accuracy_imbalance_difference' %in% names(plot.df)))
- plot.df[,accuracy_imbalance_difference:=NA]
+if(!('accuracy_imbalance_difference' %in% names(sims.df)))
+ sims.df[,accuracy_imbalance_difference:=NA]
-unique(plot.df[,'accuracy_imbalance_difference'])
+unique(sims.df[,'accuracy_imbalance_difference'])
+change.remember.file(args$remember_file, clear=TRUE)
#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
-plot.df <- build_plot_dataset(plot.df)
+plot.df <- build_plot_dataset(sims.df)
remember(plot.df,args$name)
+set.remember.prefix(gsub("plot.df.","",args$name))
+
+remember(median(sims.df$cor.xz),'med.cor.xz')
+remember(median(sims.df$accuracy),'med.accuracy')
+remember(median(sims.df$accuracy.y0),'med.accuracy.y0')
+remember(median(sims.df$accuracy.y1),'med.accuracy.y1')
+remember(median(sims.df$fpr),'med.fpr')
+remember(median(sims.df$fpr.y0),'med.fpr.y0')
+remember(median(sims.df$fpr.y1),'med.fpr.y1')
+remember(median(sims.df$fnr),'med.fnr')
+remember(median(sims.df$fnr.y0),'med.fnr.y0')
+remember(median(sims.df$fnr.y1),'med.fnr.y1')
+
+remember(median(sims.df$cor.resid.w_pred),'cor.resid.w_pred')
+
#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
## ## ## df[gmm.ER_pval<0.05]
--- /dev/null
+source("RemembR/R/RemembeR.R")
+library(arrow)
+library(data.table)
+library(ggplot2)
+library(filelock)
+library(argparser)
+
+parser <- arg_parser("Simulate data and fit corrected models.")
+parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
+args <- parse_args(parser)
+source("summarize_estimator.R")
+
+build_plot_dataset <- function(df){
+
+ x.true <- summarize.estimator(df, 'true','x')
+
+ z.true <- summarize.estimator(df, 'true','z')
+
+ x.loa0.feasible <- summarize.estimator(df, 'loa0.feasible','x')
+
+ z.loa0.feasible <- summarize.estimator(df,'loa0.feasible','z')
+
+ x.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'x')
+
+ z.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'z')
+
+ x.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'x')
+
+ z.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'z')
+
+ x.loco.mle <- summarize.estimator(df, 'loco.mle', 'x')
+
+ z.loco.mle <- summarize.estimator(df, 'loco.mle', 'z')
+
+
+ accuracy <- df[,mean(accuracy)]
+ plot.df <- rbindlist(list(x.true,z.true,x.loa0.feasible,z.loa0.feasible,x.loa0.mle,z.loa0.mle,x.loco.feasible, z.loco.feasible, z.loco.mle, x.loco.mle),use.names=T)
+ plot.df[,accuracy := accuracy]
+ plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+ return(plot.df)
+}
+
+
+plot.df <- read_feather(args$infile)
+print(unique(plot.df$N))
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(plot.df)))
+ plot.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+ plot.df[,accuracy_imbalance_difference:=NA]
+
+unique(plot.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(plot.df)
+
+change.remember.file("remember_irr.RDS",clear=TRUE)
+
+remember(plot.df,args$name)
--- /dev/null
+source("RemembR/R/RemembeR.R")
+library(arrow)
+library(data.table)
+library(ggplot2)
+library(filelock)
+library(argparser)
+
+parser <- arg_parser("Simulate data and fit corrected models.")
+parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
+args <- parse_args(parser)
+source("summarize_estimator.R")
+
+build_plot_dataset <- function(df){
+
+ x.true <- summarize.estimator(df, 'true','x')
+
+ z.true <- summarize.estimator(df, 'true','z')
+
+ x.loa0.feasible <- summarize.estimator(df, 'loa0.feasible','x')
+
+ z.loa0.feasible <- summarize.estimator(df,'loa0.feasible','z')
+
+ x.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'x')
+
+ z.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'z')
+
+ x.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'x')
+
+ z.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'z')
+
+ x.loco.mle <- summarize.estimator(df, 'loco.mle', 'x')
+
+ z.loco.mle <- summarize.estimator(df, 'loco.mle', 'z')
+
+ ## x.mle <- summarize.estimator(df, 'mle', 'x')
+
+ ## z.mle <- summarize.estimator(df, 'mle', 'z')
+
+ accuracy <- df[,mean(accuracy)]
+ plot.df <- rbindlist(list(x.true,z.true,x.loa0.feasible,z.loa0.feasible,x.loa0.mle,z.loa0.mle,x.loco.feasible, z.loco.feasible, x.loco.mle, z.loco.mle),use.names=T)
+ plot.df[,accuracy := accuracy]
+ plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+ return(plot.df)
+}
+
+
+plot.df <- read_feather(args$infile)
+print(unique(plot.df$N))
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(plot.df)))
+ plot.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+ plot.df[,accuracy_imbalance_difference:=NA]
+
+unique(plot.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(plot.df)
+change.remember.file("remember_irr.RDS",clear=TRUE)
+remember(plot.df,args$name)
+
+#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
+
+## ## ## df[gmm.ER_pval<0.05]
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle),
+## var.est = var(Bxy.est.mle),
+## N.sims = .N,
+## variable='z',
+## method='Bespoke MLE'
+## ),
+## by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')]
+
+## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+
+## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle),
+## var.est = var(Bzy.est.mle),
+## N.sims = .N,
+## variable='z',
+## method='Bespoke MLE'
+## ),
+## by=c("N","m",'y_explained_variance','Bzx')]
+
+## plot.df <- z.mle.plot
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+## N=factor(N),
+## m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=0.2),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
+
+## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
+
+## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
#SBATCH --partition=compute-bigmem
## Resources
#SBATCH --nodes=1
-## Walltime (12 hours)
-#SBATCH --time=1:00:00
+## Walltime (4 hours)
+#SBATCH --time=4:00:00
## Memory per node
-#SBATCH --mem=8G
+#SBATCH --mem=4G
#SBATCH --cpus-per-task=1
#SBATCH --ntasks-per-node=1
#SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
#SBATCH --output=simulation_jobs/%A_%a.out
#SBATCH --error=simulation_jobs/%A_%a.err
+source ~/.bashrc
TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
TASK_CALL=$(sed -n ${TASK_NUM}p $2)
accuracy <- df[,mean(w_pred==y)]
result <- append(result, list(accuracy=accuracy))
+ error.cor.x <- cor(df$x, df$w - df$x)
+ result <- append(result, list(error.cor.x = error.cor.x))
+ model.null <- glm(y~1, data=df,family=binomial(link='logit'))
(model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))
+ (lik.ratio <- exp(logLik(model.true) - logLik(model.null)))
+
true.ci.Bxy <- confint(model.true)['x',]
true.ci.Bzy <- confint(model.true)['z',]
+
+ result <- append(result, list(lik.ratio=lik.ratio))
+
result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
Bzy.est.true=coef(model.true)['z'],
Bxy.ci.upper.true = true.ci.Bxy[2],
run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=NULL, truth_formula=NULL){
accuracy <- df[,mean(w_pred==x)]
- result <- append(result, list(accuracy=accuracy))
-
+ accuracy.y0 <- df[y<=0,mean(w_pred==x)]
+ accuracy.y1 <- df[y>=0,mean(w_pred==x)]
+ cor.y.xi <- cor(df$x - df$w_pred, df$y)
+
+ fnr <- df[w_pred==0,mean(w_pred!=x)]
+ fnr.y0 <- df[(w_pred==0) & (y<=0),mean(w_pred!=x)]
+ fnr.y1 <- df[(w_pred==0) & (y>=0),mean(w_pred!=x)]
+
+ fpr <- df[w_pred==1,mean(w_pred!=x)]
+ fpr.y0 <- df[(w_pred==1) & (y<=0),mean(w_pred!=x)]
+ fpr.y1 <- df[(w_pred==1) & (y>=0),mean(w_pred!=x)]
+ cor.resid.w_pred <- cor(resid(lm(y~x+z,df)),df$w_pred)
+
+ result <- append(result, list(accuracy=accuracy,
+ accuracy.y0=accuracy.y0,
+ accuracy.y1=accuracy.y1,
+ cor.y.xi=cor.y.xi,
+ fnr=fnr,
+ fnr.y0=fnr.y0,
+ fnr.y1=fnr.y1,
+ fpr=fpr,
+ fpr.y0=fpr.y0,
+ fpr.y1=fpr.y1,
+ cor.resid.w_pred=cor.resid.w_pred
+ ))
+
+ result <- append(result, list(cor.xz=cor(df$x,df$z)))
(model.true <- lm(y ~ x + z, data=df))
true.ci.Bxy <- confint(model.true)['x',]
true.ci.Bzy <- confint(model.true)['z',]
--- /dev/null
+
+summarize.estimator <- function(df, suffix='naive', coefname='x'){
+
+ part <- df[,c('N',
+ 'm',
+ 'Bxy',
+ paste0('B',coefname,'y.est.',suffix),
+ paste0('B',coefname,'y.ci.lower.',suffix),
+ paste0('B',coefname,'y.ci.upper.',suffix),
+ 'y_explained_variance',
+ 'Bzx',
+ 'Bzy',
+ 'accuracy_imbalance_difference'
+ ),
+ with=FALSE]
+
+
+ true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+ zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+ bias <- part[[paste0('B',coefname,'y')]] - part[[paste0('B',coefname,'y.est.',suffix)]]
+ sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+ part <- part[,':='(true.in.ci = true.in.ci,
+ zero.in.ci = zero.in.ci,
+ bias=bias,
+ sign.correct =sign.correct)]
+
+ part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+ mean.bias = mean(bias),
+ mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+ est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.975,na.rm=T),
+ est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.025,na.rm=T),
+ mean.ci.upper = mean(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]]),
+ mean.ci.lower = mean(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]]),
+ ci.upper.975 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.975,na.rm=T),
+ ci.upper.025 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.025,na.rm=T),
+ ci.lower.975 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.975,na.rm=T),
+ ci.lower.025 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.025,na.rm=T),
+ N.ci.is.NA = sum(is.na(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]])),
+ N.sims = .N,
+ p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+ variable=coefname,
+ method=suffix
+ ),
+ by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
+ ]
+
+ return(part.plot)
+}
projects / ml_measurement_error_public.git / commitdiff