civil_comments/design_example.R

   1 library(data.table)
   2 library(MASS)
   3
   4 scores <- fread("perspective_scores.csv")
   5 scores <- scores[,id:=as.character(id)]
   6
   7 df <- fread("all_data.csv")
   8
   9 # only use the data that has identity annotations
  10 df <- df[identity_annotator_count > 0]
  11
  12 (df[!(df$id %in% scores$id)])
  13
  14 df <- df[scores,on='id',nomatch=NULL]
  15
  16 ## how accurate are the classifiers?
  17
  18 ## the API claims that these scores are "probabilities"
  19 ## say we care about the model of the classification, not the probability
  20
  21 F1 <- function(y, predictions){
  22     tp <- sum( (predictions == y) & (predictions==1))
  23     fn <- sum( (predictions != y) & (predictions!=1))
  24     fp <- sum( (predictions != y) & (predictions==1))
  25     precision <- tp / (tp + fp)
  26     recall <- tp / (tp + fn)
  27     return (2 * precision * recall ) / (precision + recall)
  28 }
  29
  30 df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
  31           insult_pred = insult_prob >= 0.5,
  32           profanity_pred = profanity_prob >= 0.5,
  33           severe_toxicity_pred = severe_toxicity_prob >= 0.5,
  34           threat_pred = threat_prob >= 0.5,
  35           toxicity_pred = toxicity_prob >= 0.5,
  36           identity_attack_coded = identity_attack >= 0.5,
  37           insult_coded = insult >= 0.5,
  38           profanity_coded = obscene >= 0.5,
  39           severe_toxicity_coded = severe_toxicity >= 0.5,
  40           threat_coded = threat >= 0.5,
  41           toxicity_coded = toxicity >= 0.5
  42           )]
  43
  44
  45
  46 ## toxicity is about 93% accurate, with an f1 of 0.8
  47 ## identity_attack has high accuracy 97%, but an unfortunant f1 of 0.5.
  48 ## threat has high accuracy 99%, but a really bad looking f1 of 0.48.
  49 accuracies <- df[,.(identity_attack_acc = mean(identity_attack_pred == identity_attack_coded),
  50                     insult_pred_acc = mean(insult_pred == insult_coded),
  51                     profanity_acc = mean(profanity_pred == profanity_coded),
  52                     severe_toxicity_acc = mean(severe_toxicity_pred == severe_toxicity_coded),
  53                     theat_acc = mean(threat_pred == threat_coded),
  54                     toxicity_acc = mean(toxicity_pred == toxicity_coded))]
  55
  56 f1s <- df[,.(identity_attack_f1 = F1(identity_attack_coded,identity_attack_pred),
  57                     insult_f1 = F1(insult_coded,insult_pred),
  58                     profanity_f1 = F1(profanity_coded,profanity_pred),
  59                     severe_toxicity_f1 = F1(severe_toxicity_coded,severe_toxicity_pred),
  60                     theat_f1 = F1(threat_coded,threat_pred),
  61                     toxicity_f1 = F1(toxicity_coded,toxicity_pred))]
  62
  63 positive_cases <- df[,.(identity_attacks = sum(identity_attack_coded),
  64                         insults = sum(insult_coded),
  65                         profanities = sum(profanity_coded),
  66                         severe_toxic_comments = sum(severe_toxicity_coded),
  67                         threats = sum(threat_coded),
  68                         toxic_comments = sum(toxicity_coded))]
  69
  70 ## there are 50,000 toxic comments, 13000 identity attacks, 30000 insults, 3000 profanities, 8 severe toxic, and 1000 threats.
  71
  72 proportions_cases <- df[,.(prop_identity = mean(identity_attack_coded),
  73                            prop_insults = mean(insult_coded),
  74                            prop_profanity = mean(profanity_coded),
  75                            prop_severe = mean(severe_toxicity_coded),
  76                            prop_threats = mean(threat_coded),
  77                            prop_toxic = mean(toxicity_coded))]
  78
  79 ## at 11% of comments, "toxicity" seems not so badly skewed. Try toxicity first, and if it doesn't work out try insults.
  80
  81 ## now look for an example where differential error affects an identity, or a reaction.
  82 df <- df[,":="(identity_error = identity_attack_coded - identity_attack_pred,
  83                insult_error = insult_coded - insult_pred,
  84                profanity_error = profanity_coded - profanity_pred,
  85                severe_toxic_error = severe_toxicity_coded - severe_toxicity_pred,
  86                threat_error = threat_coded - threat_pred,
  87                toxicity_error = toxicity_coded - toxicity_pred)]
  88
  89 ## what's correlated with toxicity_error ?
  90 df <- df[,approved := rating == "approved"]
  91
  92 cortab <- cor(df[,.(toxicity_error,
  93                     identity_error,
  94                     toxicity_coded,
  95                     funny,
  96                     approved,
  97                     sad,
  98                     wow,
  99                     likes,
 100                     disagree,
 101                     male,
 102                     female,
 103                     transgender,
 104                     other_gender,
 105                     heterosexual,
 106                     bisexual,
 107                     other_sexual_orientation,
 108                     christian,
 109                     jewish,
 110                     hindu,
 111                     buddhist,
 112                     atheist,
 113                     other_religion,
 114                     black,
 115                     white,
 116                     asian,
 117                     latino,
 118                     other_race_or_ethnicity,
 119                     physical_disability,
 120                     intellectual_or_learning_disability,
 121                     psychiatric_or_mental_illness,
 122                     other_disability)])
 123
 124 ## toxicity error is weakly correlated pearson's R = 0.1 with both "white" and "black".
 125
 126 ## compare regressions with "white" or "black" as the outcome and "toxicity_coded" or "toxicity_pred" as a predictor.
 127 ## here's a great example with presumambly non-differential error: about what identities is toxicity found humorous?
 128 ## a bunch of stars reappear when you used the ground-truth data instead of the predictions.
 129 ## pro/con of this example: will have to implement family='poisson'.
 130 ## shouldn't be that bad though haha.
 131 cortab['toxicity_error',]
 132 cortab['toxicity_error','funny']
 133 cortab['toxicity_coded',]
 134 cortab['identity_error',]
 135 cortab['white',]
 136
 137 glm(white ~ toxicity_coded + psychiatric_or_mental_illness, data = df, family=binomial(link='logit'))
 138
 139 glm(white ~ toxicity_pred + psychiatric_or_mental_illness, data = df, family=binomial(link='logit'))
 140
 141 m1 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_coded, data = df)
 142
 143 m2 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_pred, data = df)
 144
 145 m3 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity, data = df)
 146
 147
 148 glm(white ~ disagree, data = df, family=binomial(link='logit'))
 149
 150 ## example with differential error
 151
 152 glm(white ~ toxicity_coded + toxicity_error, data=df,family=binomial(link='logit'))
 153
 154 glm(toxicity_coded ~ white, data = df, family=binomial(link='logit'))
 155
 156 glm(toxicity_pred ~ white, data = df, family=binomial(link='logit'))