civil_comments/02_iv_example.R

   1 source("../simulations/RemembR/R/RemembeR.R")
   2 change.remember.file("iv_perspective_example.RDS")
   3
   4 source('load_perspective_data.R')
   5 source("../simulations/measerr_methods.R")
   6
   7 remember(accuracies, "civil_comments_accuracies")
   8 remember(f1s, "civil_comments_f1s")
   9 remember(positive_cases, "civil_comments_positive_cases")
  10 remember(proportions_cases, "civil_comments_proportions_cases")
  11 remember(cortab, "civil_comments_cortab")
  12 remember(nrow(df), 'n.annotated.comments')
  13 # for reproducibility
  14 set.seed(1)
  15
  16 ## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
  17
  18 compare_iv_models <-function(pred_formula, outcome_formula, proxy_formula, truth_formula, df, sample.prop, sample.size, remember_prefix){
  19
  20     if(is.null(sample.prop)){
  21         sample.prop <- sample.size / nrow(df)
  22     }
  23     if(is.null(sample.size)){
  24         sample.size <- nrow(df) * sample.prop
  25     }
  26
  27     remember(sample.size, paste0(remember_prefix, "sample.size"))
  28     remember(sample.prop, paste0(remember_prefix, "sample.prop"))
  29     remember(pred_formula, paste0(remember_prefix, "pred_formula"))
  30     remember(outcome_formula, paste0(remember_prefix, 'outcome_formula'))
  31     remember(proxy_formula, paste0(remember_prefix, 'proxy_formula'))
  32     remember(truth_formula, paste0(remember_prefix, 'truth_formula'))
  33
  34     pred_model <- glm(pred_formula, df, family=binomial(link='logit'))
  35     remember(coef(pred_model), paste0(remember_prefix, "coef_pred_model"))
  36     remember(diag(vcov((pred_model))), paste0(remember_prefix, "se_pred_model"))
  37
  38     coder_model <- glm(outcome_formula, df, family=binomial(link='logit'))
  39     remember(coef(coder_model), paste0(remember_prefix, "coef_coder_model"))
  40     remember(diag(vcov((coder_model))), paste0(remember_prefix, "se_coder_model"))
  41
  42     df_measerr_method <- copy(df)[sample(1:.N, sample.size), toxicity_coded_1 := toxicity_coded]
  43     df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
  44     sample_model <- glm(outcome_formula, df_measerr_method, family=binomial(link='logit'))
  45     remember(coef(sample_model), paste0(remember_prefix, "coef_sample_model"))
  46     remember(diag(vcov((sample_model))), paste0(remember_prefix, "se_sample_model"))
  47
  48     measerr_model <- measerr_mle(df_measerr_method, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula=proxy_formula, proxy_family=binomial(link='logit'),truth_formula=truth_formula, truth_family=binomial(link='logit'))
  49
  50     inv_hessian = solve(measerr_model$hessian)
  51     stderr = diag(inv_hessian)
  52     remember(stderr, paste0(remember_prefix, "measerr_model_stderr"))
  53     remember(measerr_model$par, paste0(remember_prefix, "measerr_model_par"))
  54 }
  55
  56 ## print("running first iv example")
  57
  58 ## sample.prop <- 0.05
  59
  60 ## compare_iv_models(white ~ toxicity_pred*funny,
  61 ##                   outcome_formula = white ~ toxicity_coded*funny,
  62 ##                   proxy_formula = toxicity_pred ~ toxicity_coded*funny*white,
  63 ##                   truth_formula = toxicity_coded ~ 1,
  64 ##                   df=df,
  65 ##                   sample.prop=sample.prop,
  66 ##                   remember_prefix='cc_ex_tox.funny.white')
  67
  68
  69
  70 pred_formula <- race_disclosed ~ likes * toxicity_pred
  71 outcome_formula <- race_disclosed ~ likes * toxicity_coded
  72 proxy_formula <- toxicity_pred ~ toxicity_coded * race_disclosed * likes
  73 truth_formula <- toxicity_coded ~ 1
  74
  75 print("running first example")
  76
  77 compare_iv_models(pred_formula = pred_formula,
  78                   outcome_formula = outcome_formula,
  79                   proxy_formula = proxy_formula,
  80                   truth_formula = truth_formula,
  81                   df=df,
  82                   sample.prop=0.01,
  83                   sample.size=NULL,
  84                   remember_prefix='cc_ex_tox.likes.race_disclosed')
  85
  86 print("running second example")
  87
  88 compare_iv_models(pred_formula = pred_formula,
  89                   outcome_formula = outcome_formula,
  90                   proxy_formula = proxy_formula,
  91                   truth_formula = truth_formula,
  92                   df=df,
  93                   sample.prop=NULL,
  94                   sample.size=10000,
  95                   remember_prefix='cc_ex_tox.likes.race_disclosed.medsamp')
  96
  97 print("running third example")
  98
  99 compare_iv_models(pred_formula = race_disclosed ~ likes * toxicity_pred,
 100                   outcome_formula = race_disclosed ~ likes * toxicity_coded,
 101                   proxy_formula = toxicity_pred ~ toxicity_coded + race_disclosed,
 102                   truth_formula = toxicity_coded ~ 1,
 103                   df=df,
 104                   sample.prop=0.05,
 105                   sample.size=NULL,
 106                   remember_prefix='cc_ex_tox.likes.race_disclosed.largesamp')
 107