civil_comments/01_dv_example.R

   1 source('load_perspective_data.R')
   2 source("../simulations/measerr_methods.R")
   3 source("../simulations/RemembR/R/RemembeR.R")
   4
   5 change.remember.file("dv_perspective_example.RDS")
   6 remember(accuracies, "civil_comments_accuracies")
   7 remember(f1s, "civil_comments_f1s")
   8 remember(positive_cases, "civil_comments_positive_cases")
   9 remember(proportions_cases, "civil_comments_proportions_cases")
  10 remember(cortab, "civil_comments_cortab")
  11 remember(nrow(df), 'n.annotated.comments')
  12 # for reproducibility
  13 set.seed(111)
  14
  15 ## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
  16
  17 compare_dv_models <-function(pred_formula, outcome_formula, proxy_formula, df, sample.prop, sample.size, remember_prefix){
  18     if(is.null(sample.prop)){
  19         sample.prop <- sample.size / nrow(df)
  20     }
  21     if(is.null(sample.size)){
  22         sample.size <- nrow(df) * sample.prop
  23     }
  24
  25     pred_model <- glm(pred_formula, df, family=binomial(link='logit'))
  26
  27     remember(sample.size, paste0(remember_prefix, "sample.size"))
  28     remember(sample.prop, paste0(remember_prefix, "sample.prop"))
  29     remember(pred_formula, paste0(remember_prefix, "pred_formula"))
  30     remember(outcome_formula, paste0(remember_prefix, 'outcome_formula'))
  31     remember(proxy_formula, paste0(remember_prefix, 'proxy_formula'))
  32
  33     remember(coef(pred_model), paste0(remember_prefix, "coef_pred_model"))
  34     remember(diag(vcov((pred_model))), paste0(remember_prefix, "se_pred_model"))
  35
  36     coder_model <- glm(outcome_formula, df, family=binomial(link='logit'))
  37     remember(coef(coder_model), paste0(remember_prefix, "coef_coder_model"))
  38     remember(diag(vcov((coder_model))), paste0(remember_prefix, "se_coder_model"))
  39
  40     df_measerr_method <- copy(df)[sample(1:.N, sample.size), toxicity_coded_1 := toxicity_coded]
  41     df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
  42     sample_model <- glm(outcome_formula, df_measerr_method, family=binomial(link='logit'))
  43     remember(coef(sample_model), paste0(remember_prefix, "coef_sample_model"))
  44     remember(diag(vcov((sample_model))), paste0(remember_prefix, "se_sample_model"))
  45
  46     measerr_model <- measerr_mle_dv(df_measerr_method, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula=proxy_formula, proxy_family=binomial(link='logit'))
  47
  48     inv_hessian = solve(measerr_model$hessian)
  49     stderr = diag(inv_hessian)
  50     remember(stderr, paste0(remember_prefix, "measerr_model_stderr"))
  51     remember(measerr_model$par, paste0(remember_prefix, "measerr_model_par"))
  52 }
  53
  54 print("running first example")
  55
  56 pred_formula = toxicity_pred ~ likes + race_disclosed
  57 outcome_formula = toxicity_coded ~ likes + race_disclosed
  58 proxy_formula = toxicity_pred ~ toxicity_coded*race_disclosed*likes
  59
  60 compare_dv_models(pred_formula = pred_formula,
  61                   outcome_formula = outcome_formula,
  62                   proxy_formula = proxy_formula,
  63                   df=df,
  64                   sample.prop=0.01,
  65                   sample.size=NULL,
  66                   remember_prefix='cc_ex_tox.likes.race_disclosed')
  67
  68
  69 print("running second example")
  70
  71 compare_dv_models(pred_formula = pred_formula,
  72                   outcome_formula = outcome_formula,
  73                   proxy_formula = proxy_formula,
  74                   df=df,
  75                   sample.size=10000,
  76                   sample.prop=NULL,
  77                   remember_prefix='cc_ex_tox.likes.race_disclosed.medsamp')
  78
  79
  80 print("running third example")
  81
  82 compare_dv_models(pred_formula = pred_formula,
  83                   outcome_formula = outcome_formula,
  84                   proxy_formula = proxy_formula,
  85                   df=df,
  86                   sample.prop=0.05,
  87                   sample.size=NULL,
  88                   remember_prefix='cc_ex_tox.likes.race_disclosed.largesamp')
  89