civil_comments/design_example.R

   1 set.seed(1111)
   2 source('load_perspective_data.R')
   3 ## how accurate are the classifiers?
   4
   5 ## the API claims that these scores are "probabilities"
   6 ## say we care about the model of the classification, not the probability
   7
   8 ## toxicity error is weakly correlated pearson's R = 0.1 with both "white" and "black".
   9
  10 ## compare regressions with "white" or "black" as the outcome and "toxicity_coded" or "toxicity_pred" as a predictor.
  11 ## here's a great example with presumambly non-differential error: about what identities is toxicity found humorous?
  12 ## a bunch of stars reappear when you used the ground-truth data instead of the predictions.
  13 ## pro/con of this example: will have to implement family='poisson'.
  14 ## shouldn't be that bad though haha.
  15 cortab['toxicity_error',]
  16 cortab['toxicity_error','funny']
  17 cortab['toxicity_coded',]
  18 cortab['identity_error',]
  19 cortab['white',]
  20
  21 ## here's a simple example, is P(white | toxic and mentally ill) > P(white | toxic or mentally ill). Are people who discuss their mental illness in a toxic way more likely to be white compared to those who just talk about their mental illness or are toxic?
  22 summary(glm(white ~ toxicity_coded*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
  23
  24 summary(glm(white ~ toxicity_pred*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
  25
  26 summary(glm(white ~ toxicity_coded*male, data = df, family=binomial(link='logit')))
  27
  28 summary(glm(white ~ toxicity_pred*male, data = df, family=binomial(link='logit')))
  29
  30 summary(glm(toxicity_coded ~ white*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
  31
  32 summary(glm(toxicity_pred ~ white*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
  33
  34
  35 ## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
  36
  37 summary(glm(toxicity_pred ~ funny*white, data=df, family=binomial(link='logit')))
  38 summary(glm(toxicity_coded ~ funny*white, data=df, family=binomial(link='logit')))
  39
  40 source("../simulations/measerr_methods.R")
  41
  42 saved_model_file <- "measerr_model_tox.eq.funny.cross.white.RDS"
  43 overwrite_model <- TRUE
  44
  45 # it works so far with a 20% and 15% sample. Smaller is better. let's try a 10% sample again. It didn't work out. We'll go forward with a 15% sample.
  46 df_measerr_method <- copy(df)[sample(1:.N, 0.05 * .N), toxicity_coded_1 := toxicity_coded]
  47 df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
  48 summary(glm(toxicity_coded ~ funny*white, data=df_measerr_method[!is.na(toxicity_coded)], family=binomial(link='logit')))
  49
  50 if(!file.exists(saved_model_file) || (overwrite_model == TRUE)){
  51     measerr_model <- measerr_mle_dv(df_measerr_method,toxicity_coded ~ funny*white,outcome_family=binomial(link='logit'), proxy_formula=toxicity_pred ~ toxicity_coded*funny*white)
  52 saveRDS(measerr_model, saved_model_file)
  53 } else {
  54     measerr_model <- readRDS(saved_model_file)
  55 }
  56
  57 inv_hessian <- solve(measerr_model$hessian)
  58 se <- diag(inv_hessian)
  59
  60 lm2 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_pred, data = df)
  61 m3 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity, data = df)
  62
  63
  64 glm(white ~ disagree, data = df, family=binomial(link='logit'))
  65
  66 ## example with differential error
  67
  68 glm(white ~ toxicity_coded + toxicity_error, data=df,family=binomial(link='logit'))
  69
  70 glm(toxicity_coded ~ white, data = df, family=binomial(link='logit'))
  71
  72 glm(toxicity_pred ~ white, data = df, family=binomial(link='logit'))