From: Nathan TeBlunthuis Date: Fri, 7 Oct 2022 17:42:50 +0000 (-0700) Subject: Update stuff. X-Git-Url: https://code.communitydata.science/ml_measurement_error_public.git/commitdiff_plain/b52b4f7daaba8a877b041ddb24c8f36b466ddc5b?ds=sidebyside;hp=979dc14b6861ae31f00d56392fd5b8cf69f17333 Update stuff. --- diff --git a/simulations/02_indep_differential.R b/simulations/02_indep_differential.R index c6907d3..bcfad65 100644 --- a/simulations/02_indep_differential.R +++ b/simulations/02_indep_differential.R @@ -104,8 +104,9 @@ simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0. ## print(mean(df[z==1]$x == df[z==1]$w_pred)) ## print(mean(df$w_pred == df$x)) - odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(scale(df[x==1]$y))) - odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(scale(df[x==0]$y))) + resids <- resid(lm(y~x + z)) + odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(resids[x==1])) + odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(resids[x==0])) ## acc.x0 <- p.correct[df[,x==0]] ## acc.x1 <- p.correct[df[,x==1]] @@ -115,8 +116,7 @@ simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0. df[,w_pred := as.integer(w > 0.5)] - print(mean(df[z==0]$x == df[z==0]$w_pred)) - print(mean(df[z==1]$x == df[z==1]$w_pred)) + print(mean(df$w_pred == df$x)) print(mean(df[y>=0]$w_pred == df[y>=0]$x)) print(mean(df[y<=0]$w_pred == df[y<=0]$x)) @@ -124,7 +124,7 @@ simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0. } parser <- arg_parser("Simulate data and fit corrected models") -parser <- add_argument(parser, "--N", default=1000, help="number of observations of w") +parser <- add_argument(parser, "--N", default=5000, help="number of observations of w") parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations") parser <- add_argument(parser, "--seed", default=51, help='seed for the rng') parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather') @@ -136,7 +136,7 @@ parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3) parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3) parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z") parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x") -parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.75) +parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-1) parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z") args <- parse_args(parser) diff --git a/simulations/03_depvar.R b/simulations/03_depvar.R index a2d88e0..79a516f 100644 --- a/simulations/03_depvar.R +++ b/simulations/03_depvar.R @@ -31,7 +31,8 @@ source("simulation_base.R") ## one way to do it is by adding correlation to x.obs and y that isn't in w. ## in other words, the model is missing an important feature of x.obs that's related to y. -simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73){ +simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, log.likelihood.gain = 0.1){ + set.seed(seed) set.seed(seed) # make w and y dependent @@ -41,8 +42,6 @@ simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73){ ystar <- Bzy * z + Bxy * x + B0 y <- rbinom(N,1,plogis(ystar)) - # glm(y ~ x + z, family="binomial") - df <- data.table(x=x,y=y,ystar=ystar,z=z) if(m < N){ @@ -66,7 +65,7 @@ simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73){ } parser <- arg_parser("Simulate data and fit corrected models") -parser <- add_argument(parser, "--N", default=1000, help="number of observations of w") +parser <- add_argument(parser, "--N", default=10000, help="number of observations of w") parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations") parser <- add_argument(parser, "--seed", default=17, help='seed for the rng') parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather') @@ -74,8 +73,8 @@ parser <- add_argument(parser, "--y_explained_variance", help='what proportion o parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72) ## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75) ## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75) -parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3) -parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3) +parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.01) +parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.01) parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z") parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y") diff --git a/simulations/Makefile b/simulations/Makefile index 44910cb..af54727 100644 --- a/simulations/Makefile +++ b/simulations/Makefile @@ -2,19 +2,20 @@ SHELL=bash Ns=[1000, 2000, 4000] -ms=[200, 400, 800] +ms=[100, 200, 400, 800] seeds=[$(shell seq -s, 1 250)] explained_variances=[0.1] all:remembr.RDS remember_irr.RDS +supplement: remember_robustness_misspec.RDS -srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1 +srun=sbatch --wait --verbose run_job.sbatch joblists:example_1_jobs example_2_jobs example_3_jobs # test_true_z_jobs: test_true_z.R simulation_base.R -# grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb +# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb # test_true_z.feather: test_true_z_jobs # rm -f test_true_z.feather @@ -22,45 +23,45 @@ joblists:example_1_jobs example_2_jobs example_3_jobs # sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs -example_1_jobs: 01_two_covariates.R simulation_base.R - grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs +example_1_jobs: 01_two_covariates.R simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.3]}' --outfile example_1_jobs example_1.feather: example_1_jobs rm -f example_1.feather sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs # sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs -example_2_jobs: 02_indep_differential.R simulation_base.R - grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y*z*x"]}' --outfile example_2_jobs +example_2_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y*z*x"]}' --outfile example_2_jobs example_2.feather: example_2_jobs rm -f example_2.feather - sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs + sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs # sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs # example_2_B_jobs: example_2_B.R -# grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs +# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs # example_2_B.feather: example_2_B_jobs # rm -f example_2_B.feather # sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs -example_3_jobs: 03_depvar.R simulation_base.R - grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs +example_3_jobs: 03_depvar.R simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "Bxy":[0.01],"Bzy":[-0.01],"seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs example_3.feather: example_3_jobs rm -f example_3.feather sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs -example_4_jobs: 04_depvar_differential.R simulation_base.R - grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "y_explained_variance":${explained_variances}}' --outfile example_4_jobs +example_4_jobs: 04_depvar_differential.R simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"Bxy":[0.01],"Bzy":[-0.01],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "y_explained_variance":${explained_variances}}' --outfile example_4_jobs example_4.feather: example_4_jobs rm -f example_4.feather sbatch --wait --verbose --array=1-$(shell cat example_4_jobs | wc -l) run_simulation.sbatch 0 example_4_jobs -remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R +remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R summarize_estimator.R rm -f remembr.RDS ${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1" ${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2" @@ -73,25 +74,51 @@ irr_ms = ${ms} irr_seeds=${seeds} irr_explained_variances=${explained_variances} -example_5_jobs: 05_irr_indep.R irr_simulation_base.R - grid_sweep.py --command "Rscript 05_irr_indep.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_5.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_5_jobs +example_5_jobs: 05_irr_indep.R irr_simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 05_irr_indep.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_5.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_5_jobs example_5.feather:example_5_jobs rm -f example_5.feather sbatch --wait --verbose --array=1-$(shell cat example_5_jobs | wc -l) run_simulation.sbatch 0 example_5_jobs -example_6_jobs: 06_irr_dv.R irr_dv_simulation_base.R - grid_sweep.py --command "Rscript 06_irr_dv.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_6.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_6_jobs +example_6_jobs: 06_irr_dv.R irr_dv_simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 06_irr_dv.R" --arg_dict '{"N":${irr_Ns},"m":${irr_ms}, "seed":${irr_seeds}, "outfile":["example_6.feather"], "y_explained_variance":${irr_explained_variances}}' --outfile example_6_jobs example_6.feather:example_6_jobs rm -f example_6.feather sbatch --wait --verbose --array=1-$(shell cat example_6_jobs | wc -l) run_simulation.sbatch 0 example_6_jobs -remember_irr.RDS: example_5.feather example_6.feather plot_irr_example.R plot_irr_dv_example.R +remember_irr.RDS: example_5.feather example_6.feather plot_irr_example.R plot_irr_dv_example.R summarize_estimator.R rm -f remember_irr.RDS - ${srun} Rscript plot_irr_example.R --infile example_5.feather --name "plot.df.example.5" - ${srun} Rscript plot_irr_dv_example.R --infile example_6.feather --name "plot.df.example.6" + sbatch --wait --verbose run_job.sbatch Rscript plot_irr_example.R --infile example_5.feather --name "plot.df.example.5" + sbatch --wait --verbose run_job.sbatch Rscript plot_irr_dv_example.R --infile example_6.feather --name "plot.df.example.6" + + + +robustness_1_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py + sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs + + + +robustness_1.feather: robustness_1_jobs + rm -f robustness_1.feather + sbatch --wait --verbose --array=1-$(shell cat robustness_1_jobs | wc -l) run_simulation.sbatch 0 robustness_1_jobs + +robustness_1_dv_jobs: simulation_base.R 04_depvar_differential.R grid_sweep.py + ${srun} bash -c "source ~/.bashrc && grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict \"{'N':${Ns},'m':${ms}, 'seed':${seeds}, 'outfile':['robustness_1_dv.feather'], 'y_explained_variance':${explained_variances}, 'proxy_formula':['w_pred~y']}\" --outfile robustness_1_dv_jobs" + + +robustness_1_dv.feather: robustness_1_dv_jobs + rm -f robustness_1_dv.feather + sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 robustness_1_dv_jobs + + +remember_robustness_misspec.RDS: robustness_1.feather robustness_1_dv.feather + rm -f remember_robustness_misspec.RDS + sbatch --wait --verbose run_job.sbatch Rscript plot_example.R --infile robustness_1.feather --name "plot.df.robustness.1" --remember-file "remember_robustness_misspec.RDS" + sbatch --wait --verbose run_job.sbatch Rscript plot_dv_example.R --infile robustness_1_dv.feather --name "plot.df.robustness.1.dv" --remember-file "remember_robustness_mispec.RDS" + clean: rm *.feather @@ -100,7 +127,7 @@ clean: # sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs # example_2_B_mecor_jobs: -# grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs +# sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs # example_2_B_mecor.feather:example_2_B_mecor.R example_2_B_mecor_jobs # rm -f example_2_B_mecor.feather @@ -109,3 +136,4 @@ clean: +.PHONY: supplement diff --git a/simulations/grid_sweep.py b/simulations/grid_sweep.py index 86312ea..7db920d 100755 --- a/simulations/grid_sweep.py +++ b/simulations/grid_sweep.py @@ -2,8 +2,13 @@ import fire from itertools import product +import pyRemembeR -def main(command, arg_dict, outfile): +def main(command, arg_dict, outfile, remember_file='remember_grid_sweep.RDS'): + remember = pyRemembeR.remember.Remember() + remember.set_file(remember_file) + remember[outfile] = arg_dict + remember.save_to_r() keys = [] values = [] diff --git a/simulations/irr_dv_simulation_base.R b/simulations/irr_dv_simulation_base.R index 3f63d7a..059473c 100644 --- a/simulations/irr_dv_simulation_base.R +++ b/simulations/irr_dv_simulation_base.R @@ -82,7 +82,7 @@ run_simulation_depvar <- function(df, result, outcome_formula = y ~ x + z, rater Bxy.ci.upper.loco.mle = ci.upper['x'], Bxy.ci.lower.loco.mle = ci.lower['x'], Bzy.ci.upper.loco.mle = ci.upper['z'], - Bzy.ci.lower.loco.mle = ci.upper['z'])) + Bzy.ci.lower.loco.mle = ci.lower['z'])) print(rater_formula) print(proxy_formula) diff --git a/simulations/irr_simulation_base.R b/simulations/irr_simulation_base.R index ebb215b..ee7112a 100644 --- a/simulations/irr_simulation_base.R +++ b/simulations/irr_simulation_base.R @@ -82,7 +82,7 @@ run_simulation <- function(df, result, outcome_formula = y ~ x + z, proxy_formul Bxy.ci.upper.loco.mle = ci.upper['x'], Bxy.ci.lower.loco.mle = ci.lower['x'], Bzy.ci.upper.loco.mle = ci.upper['z'], - Bzy.ci.lower.loco.mle = ci.upper['z'])) + Bzy.ci.lower.loco.mle = ci.lower['z'])) ## print(rater_formula) ## print(proxy_formula) diff --git a/simulations/measerr_methods.R b/simulations/measerr_methods.R index 00f1746..087c608 100644 --- a/simulations/measerr_methods.R +++ b/simulations/measerr_methods.R @@ -1,6 +1,6 @@ library(formula.tools) library(matrixStats) - +library(bbmle) ## df: dataframe to model ## outcome_formula: formula for y | x, z ## outcome_family: family for y | x, z @@ -17,7 +17,7 @@ library(matrixStats) ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y -measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){ +measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){ nll <- function(params){ df.obs <- model.frame(outcome_formula, df) @@ -98,12 +98,23 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo start <- rep(0.1,length(params)) names(start) <- params - fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + if(method=='optim'){ + fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + } else { + quoted.names <- gsub("[\\(\\)]",'',names(start)) + print(quoted.names) + text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}") + + measerr_mle_nll <- eval(parse(text=text)) + names(start) <- quoted.names + names(lower) <- quoted.names + fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B') + } return(fit) } ## Experimental, and not necessary if errors are independent. -measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){ +measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){ ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. @@ -293,14 +304,28 @@ measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rate start <- rep(0.1,length(params)) names(start) <- params - fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + + if(method=='optim'){ + fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + } else { + + quoted.names <- gsub("[\\(\\)]",'',names(start)) + print(quoted.names) + text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}") + + measerr_mle_nll <- eval(parse(text=text)) + names(start) <- quoted.names + names(lower) <- quoted.names + fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B') + } + return(fit) } -measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){ +measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){ - measrr_mle_nll <- function(params){ + measerr_mle_nll <- function(params){ df.obs <- model.frame(outcome_formula, df) proxy.variable <- all.vars(proxy_formula)[1] proxy.model.matrix <- model.matrix(proxy_formula, df) @@ -425,8 +450,21 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo lower <- c(lower, rep(-Inf, length(truth.params))) start <- rep(0.1,length(params)) names(start) <- params - - fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + + if(method=='optim'){ + fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6)) + } else { # method='mle2' + + quoted.names <- gsub("[\\(\\)]",'',names(start)) + + text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}") + + measerr_mle_nll_mle <- eval(parse(text=text)) + names(start) <- quoted.names + names(lower) <- quoted.names + fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B') + } return(fit) } + diff --git a/simulations/plot_dv_example.R b/simulations/plot_dv_example.R index 4052c38..71963b1 100644 --- a/simulations/plot_dv_example.R +++ b/simulations/plot_dv_example.R @@ -7,49 +7,51 @@ library(argparser) parser <- arg_parser("Simulate data and fit corrected models.") parser <- add_argument(parser, "--infile", default="", help="name of the file to read.") +parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.") parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.") args <- parse_args(parser) -summarize.estimator <- function(df, suffix='naive', coefname='x'){ - - part <- df[,c('N', - 'm', - 'Bxy', - paste0('B',coefname,'y.est.',suffix), - paste0('B',coefname,'y.ci.lower.',suffix), - paste0('B',coefname,'y.ci.upper.',suffix), - 'y_explained_variance', - 'Bzy' - ), - with=FALSE] +## summarize.estimator <- function(df, suffix='naive', coefname='x'){ + +## part <- df[,c('N', +## 'm', +## 'Bxy', +## paste0('B',coefname,'y.est.',suffix), +## paste0('B',coefname,'y.ci.lower.',suffix), +## paste0('B',coefname,'y.ci.upper.',suffix), +## 'y_explained_variance', +## 'Bzy' +## ), +## with=FALSE] - true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) - zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) - bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] - sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) - - part <- part[,':='(true.in.ci = true.in.ci, - zero.in.ci = zero.in.ci, - bias=bias, - sign.correct =sign.correct)] - - part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), - var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), - est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95), - est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable=coefname, - method=suffix - ), - by=c("N","m",'Bzy','y_explained_variance') - ] +## true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) +## zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) +## bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] +## sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) + +## part <- part[,':='(true.in.ci = true.in.ci, +## zero.in.ci = zero.in.ci, +## bias=bias, +## sign.correct =sign.correct)] + +## part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), +## mean.bias = mean(bias), +## mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), +## var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), +## est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95), +## est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05), +## N.sims = .N, +## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), +## variable=coefname, +## method=suffix +## ), +## by=c("N","m",'Bzy','y_explained_variance') +## ] - return(part.plot) -} +## return(part.plot) +## } +source("summarize_estimator.R") build_plot_dataset <- function(df){ @@ -82,12 +84,23 @@ build_plot_dataset <- function(df){ return(plot.df) } - -df <- read_feather(args$infile) -plot.df <- build_plot_dataset(df) +change.remember.file(args$remember_file, clear=TRUE) +sims.df <- read_feather(args$infile) +sims.df[,Bzx:=NA] +sims.df[,accuracy_imbalance_difference:=NA] +plot.df <- build_plot_dataset(sims.df) remember(plot.df,args$name) +set.remember.prefix(gsub("plot.df.","",args$name)) + +remember(median(sims.df$cor.xz),'med.cor.xz') +remember(median(sims.df$accuracy),'med.accuracy') +remember(median(sims.df$error.cor.x),'med.error.cor.x') +remember(median(sims.df$lik.ratio),'med.lik.ratio') + + + ## df[gmm.ER_pval<0.05] ## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T), diff --git a/simulations/plot_example.R b/simulations/plot_example.R index 7a853b7..8e6c477 100644 --- a/simulations/plot_example.R +++ b/simulations/plot_example.R @@ -5,52 +5,58 @@ library(ggplot2) library(filelock) library(argparser) +source("summarize_estimator.R") + + parser <- arg_parser("Simulate data and fit corrected models.") parser <- add_argument(parser, "--infile", default="", help="name of the file to read.") +parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.") parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.") args <- parse_args(parser) -summarize.estimator <- function(df, suffix='naive', coefname='x'){ - - part <- df[,c('N', - 'm', - 'Bxy', - paste0('B',coefname,'y.est.',suffix), - paste0('B',coefname,'y.ci.lower.',suffix), - paste0('B',coefname,'y.ci.upper.',suffix), - 'y_explained_variance', - 'Bzx', - 'Bzy', - 'accuracy_imbalance_difference' - ), - with=FALSE] + + +## summarize.estimator <- function(df, suffix='naive', coefname='x'){ + +## part <- df[,c('N', +## 'm', +## 'Bxy', +## paste0('B',coefname,'y.est.',suffix), +## paste0('B',coefname,'y.ci.lower.',suffix), +## paste0('B',coefname,'y.ci.upper.',suffix), +## 'y_explained_variance', +## 'Bzx', +## 'Bzy', +## 'accuracy_imbalance_difference' +## ), +## with=FALSE] - true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) - zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) - bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] - sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) - - part <- part[,':='(true.in.ci = true.in.ci, - zero.in.ci = zero.in.ci, - bias=bias, - sign.correct =sign.correct)] - - part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), - mean.bias = mean(bias), - mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), - var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), - est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T), - est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T), - N.sims = .N, - p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), - variable=coefname, - method=suffix - ), - by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference') - ] +## true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) +## zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) +## bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] +## sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) + +## part <- part[,':='(true.in.ci = true.in.ci, +## zero.in.ci = zero.in.ci, +## bias=bias, +## sign.correct =sign.correct)] + +## part.plot <- part[, .(p.true.in.ci = mean(true.in.ci), +## mean.bias = mean(bias), +## mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), +## var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), +## est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T), +## est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T), +## N.sims = .N, +## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), +## variable=coefname, +## method=suffix +## ), +## by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference') +## ] - return(part.plot) -} +## return(part.plot) +## } build_plot_dataset <- function(df){ @@ -98,24 +104,40 @@ build_plot_dataset <- function(df){ } -plot.df <- read_feather(args$infile) -print(unique(plot.df$N)) +sims.df <- read_feather(args$infile) +print(unique(sims.df$N)) # df <- df[apply(df,1,function(x) !any(is.na(x)))] -if(!('Bzx' %in% names(plot.df))) - plot.df[,Bzx:=NA] +if(!('Bzx' %in% names(sims.df))) + sims.df[,Bzx:=NA] -if(!('accuracy_imbalance_difference' %in% names(plot.df))) - plot.df[,accuracy_imbalance_difference:=NA] +if(!('accuracy_imbalance_difference' %in% names(sims.df))) + sims.df[,accuracy_imbalance_difference:=NA] -unique(plot.df[,'accuracy_imbalance_difference']) +unique(sims.df[,'accuracy_imbalance_difference']) +change.remember.file(args$remember_file, clear=TRUE) #plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700]) -plot.df <- build_plot_dataset(plot.df) +plot.df <- build_plot_dataset(sims.df) remember(plot.df,args$name) +set.remember.prefix(gsub("plot.df.","",args$name)) + +remember(median(sims.df$cor.xz),'med.cor.xz') +remember(median(sims.df$accuracy),'med.accuracy') +remember(median(sims.df$accuracy.y0),'med.accuracy.y0') +remember(median(sims.df$accuracy.y1),'med.accuracy.y1') +remember(median(sims.df$fpr),'med.fpr') +remember(median(sims.df$fpr.y0),'med.fpr.y0') +remember(median(sims.df$fpr.y1),'med.fpr.y1') +remember(median(sims.df$fnr),'med.fnr') +remember(median(sims.df$fnr.y0),'med.fnr.y0') +remember(median(sims.df$fnr.y1),'med.fnr.y1') + +remember(median(sims.df$cor.resid.w_pred),'cor.resid.w_pred') + #ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy) ## ## ## df[gmm.ER_pval<0.05] diff --git a/simulations/run_simulation.sbatch b/simulations/run_simulation.sbatch index 54f56be..9dce9ea 100644 --- a/simulations/run_simulation.sbatch +++ b/simulations/run_simulation.sbatch @@ -5,15 +5,16 @@ #SBATCH --partition=compute-bigmem ## Resources #SBATCH --nodes=1 -## Walltime (12 hours) -#SBATCH --time=1:00:00 +## Walltime (4 hours) +#SBATCH --time=4:00:00 ## Memory per node -#SBATCH --mem=8G +#SBATCH --mem=4G #SBATCH --cpus-per-task=1 #SBATCH --ntasks-per-node=1 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations #SBATCH --output=simulation_jobs/%A_%a.out #SBATCH --error=simulation_jobs/%A_%a.err +source ~/.bashrc TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1)) TASK_CALL=$(sed -n ${TASK_NUM}p $2) diff --git a/simulations/simulation_base.R b/simulations/simulation_base.R index ee46ec6..27f0276 100644 --- a/simulations/simulation_base.R +++ b/simulations/simulation_base.R @@ -210,11 +210,19 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu accuracy <- df[,mean(w_pred==y)] result <- append(result, list(accuracy=accuracy)) + error.cor.x <- cor(df$x, df$w - df$x) + result <- append(result, list(error.cor.x = error.cor.x)) + model.null <- glm(y~1, data=df,family=binomial(link='logit')) (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit'))) + (lik.ratio <- exp(logLik(model.true) - logLik(model.null))) + true.ci.Bxy <- confint(model.true)['x',] true.ci.Bzy <- confint(model.true)['z',] + + result <- append(result, list(lik.ratio=lik.ratio)) + result <- append(result, list(Bxy.est.true=coef(model.true)['x'], Bzy.est.true=coef(model.true)['z'], Bxy.ci.upper.true = true.ci.Bxy[2], @@ -322,8 +330,33 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=NULL, truth_formula=NULL){ accuracy <- df[,mean(w_pred==x)] - result <- append(result, list(accuracy=accuracy)) - + accuracy.y0 <- df[y<=0,mean(w_pred==x)] + accuracy.y1 <- df[y>=0,mean(w_pred==x)] + cor.y.xi <- cor(df$x - df$w_pred, df$y) + + fnr <- df[w_pred==0,mean(w_pred!=x)] + fnr.y0 <- df[(w_pred==0) & (y<=0),mean(w_pred!=x)] + fnr.y1 <- df[(w_pred==0) & (y>=0),mean(w_pred!=x)] + + fpr <- df[w_pred==1,mean(w_pred!=x)] + fpr.y0 <- df[(w_pred==1) & (y<=0),mean(w_pred!=x)] + fpr.y1 <- df[(w_pred==1) & (y>=0),mean(w_pred!=x)] + cor.resid.w_pred <- cor(resid(lm(y~x+z,df)),df$w_pred) + + result <- append(result, list(accuracy=accuracy, + accuracy.y0=accuracy.y0, + accuracy.y1=accuracy.y1, + cor.y.xi=cor.y.xi, + fnr=fnr, + fnr.y0=fnr.y0, + fnr.y1=fnr.y1, + fpr=fpr, + fpr.y0=fpr.y0, + fpr.y1=fpr.y1, + cor.resid.w_pred=cor.resid.w_pred + )) + + result <- append(result, list(cor.xz=cor(df$x,df$z))) (model.true <- lm(y ~ x + z, data=df)) true.ci.Bxy <- confint(model.true)['x',] true.ci.Bzy <- confint(model.true)['z',] diff --git a/simulations/summarize_estimator.R b/simulations/summarize_estimator.R index 8199c06..e0e7622 100644 --- a/simulations/summarize_estimator.R +++ b/simulations/summarize_estimator.R @@ -13,10 +13,11 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){ 'accuracy_imbalance_difference' ), with=FALSE] - + + true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])) zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]) - bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]] + bias <- part[[paste0('B',coefname,'y')]] - part[[paste0('B',coefname,'y.est.',suffix)]] sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]])) part <- part[,':='(true.in.ci = true.in.ci, @@ -28,8 +29,15 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){ mean.bias = mean(bias), mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]), var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]), - est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T), - est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T), + est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.975,na.rm=T), + est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.025,na.rm=T), + mean.ci.upper = mean(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]]), + mean.ci.lower = mean(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]]), + ci.upper.975 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.975,na.rm=T), + ci.upper.025 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.025,na.rm=T), + ci.lower.975 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.975,na.rm=T), + ci.lower.025 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.025,na.rm=T), + N.ci.is.NA = sum(is.na(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]])), N.sims = .N, p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))), variable=coefname,