civil_comments/load_perspective_data.R

   1 library(data.table)
   2 library(MASS)
   3
   4 set.seed(1111)
   5
   6 scores <- fread("perspective_scores.csv")
   7 scores <- scores[,id:=as.character(id)]
   8
   9 df <- fread("all_data.csv")
  10
  11 # only use the data that has identity annotations
  12 df <- df[identity_annotator_count > 0]
  13
  14 (df[!(df$id %in% scores$id)])
  15
  16 df <- df[scores,on='id',nomatch=NULL]
  17
  18 df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
  19           insult_pred = insult_prob >= 0.5,
  20           profanity_pred = profanity_prob >= 0.5,
  21           severe_toxicity_pred = severe_toxicity_prob >= 0.5,
  22           threat_pred = threat_prob >= 0.5,
  23           toxicity_pred = toxicity_prob >= 0.5,
  24           identity_attack_coded = identity_attack >= 0.5,
  25           insult_coded = insult >= 0.5,
  26           profanity_coded = obscene >= 0.5,
  27           severe_toxicity_coded = severe_toxicity >= 0.5,
  28           threat_coded = threat >= 0.5,
  29           toxicity_coded = toxicity >= 0.5
  30           )]
  31
  32 gt.0.5 <- function(v) { v >= 0.5 }
  33 dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}
  34
  35 df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
  36                sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
  37                religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
  38                race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity),
  39                disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]
  40
  41 df <- df[,white:=gt.0.5(white)]
  42
  43
  44 F1 <- function(y, predictions){
  45     tp <- sum( (predictions == y) & (predictions==1))
  46     fn <- sum( (predictions != y) & (predictions!=1))
  47     fp <- sum( (predictions != y) & (predictions==1))
  48     precision <- tp / (tp + fp)
  49     recall <- tp / (tp + fn)
  50     return (2 * precision * recall ) / (precision + recall)
  51 }
  52
  53
  54 ## toxicity is about 93% accurate, with an f1 of 0.8
  55 ## identity_attack has high accuracy 97%, but an unfortunant f1 of 0.5.
  56 ## threat has high accuracy 99%, but a really bad looking f1 of 0.48.
  57 accuracies <- df[,.(identity_attack_acc = mean(identity_attack_pred == identity_attack_coded),
  58                     insult_pred_acc = mean(insult_pred == insult_coded),
  59                     profanity_acc = mean(profanity_pred == profanity_coded),
  60                     severe_toxicity_acc = mean(severe_toxicity_pred == severe_toxicity_coded),
  61                     theat_acc = mean(threat_pred == threat_coded),
  62                     toxicity_acc = mean(toxicity_pred == toxicity_coded))]
  63
  64 f1s <- df[,.(identity_attack_f1 = F1(identity_attack_coded,identity_attack_pred),
  65                     insult_f1 = F1(insult_coded,insult_pred),
  66                     profanity_f1 = F1(profanity_coded,profanity_pred),
  67                     severe_toxicity_f1 = F1(severe_toxicity_coded,severe_toxicity_pred),
  68                     theat_f1 = F1(threat_coded,threat_pred),
  69                     toxicity_f1 = F1(toxicity_coded,toxicity_pred))]
  70
  71 positive_cases <- df[,.(identity_attacks = sum(identity_attack_coded),
  72                         insults = sum(insult_coded),
  73                         profanities = sum(profanity_coded),
  74                         severe_toxic_comments = sum(severe_toxicity_coded),
  75                         threats = sum(threat_coded),
  76                         toxic_comments = sum(toxicity_coded))]
  77
  78 ## there are 50,000 toxic comments, 13000 identity attacks, 30000 insults, 3000 profanities, 8 severe toxic, and 1000 threats.
  79
  80 proportions_cases <- df[,.(prop_identity = mean(identity_attack_coded),
  81                            prop_insults = mean(insult_coded),
  82                            prop_profanity = mean(profanity_coded),
  83                            prop_severe = mean(severe_toxicity_coded),
  84                            prop_threats = mean(threat_coded),
  85                            prop_toxic = mean(toxicity_coded))]
  86
  87 ## at 11% of comments, "toxicity" seems not so badly skewed. Try toxicity first, and if it doesn't work out try insults.
  88
  89 ## now look for an example where differential error affects an identity, or a reaction.
  90 df <- df[,":="(identity_error = identity_attack_coded - identity_attack_pred,
  91                insult_error = insult_coded - insult_pred,
  92                profanity_error = profanity_coded - profanity_pred,
  93                severe_toxic_error = severe_toxicity_coded - severe_toxicity_pred,
  94                threat_error = threat_coded - threat_pred,
  95                toxicity_error = toxicity_coded - toxicity_pred)]
  96
  97 ## what's correlated with toxicity_error ?
  98 df <- df[,approved := rating == "approved"]
  99 df <- df[,white := white > 0.5]
 100
 101 cortab <- cor(df[,.(toxicity_error,
 102                     identity_error,
 103                     toxicity_coded,
 104                     funny,
 105                     approved,
 106                     sad,
 107                     wow,
 108                     likes,
 109                     disagree,
 110                     male,
 111                     female,
 112                     transgender,
 113                     other_gender,
 114                     heterosexual,
 115                     bisexual,
 116                     other_sexual_orientation,
 117                     christian,
 118                     jewish,
 119                     hindu,
 120                     buddhist,
 121                     atheist,
 122                     other_religion,
 123                     black,
 124                     white,
 125                     asian,
 126                     latino,
 127                     other_race_or_ethnicity,
 128                     physical_disability,
 129                     intellectual_or_learning_disability,
 130                     psychiatric_or_mental_illness,
 131                     other_disability,
 132                     gender_disclosed,
 133                     sexuality_disclosed,
 134                     religion_disclosed,
 135                     race_disclosed,
 136                     disability_disclosed)])