civil_comments/load_perspective_data.R

   1 library(data.table)
   2 library(MASS)
   3
   4 set.seed(1111)
   5
   6 scores <- fread("perspective_scores.csv")
   7 scores <- scores[,id:=as.character(id)]
   8
   9 df <- fread("all_data.csv")
  10
  11 # only use the data that has identity annotations
  12 df <- df[identity_annotator_count > 0]
  13
  14 (df[!(df$id %in% scores$id)])
  15
  16 df <- df[scores,on='id',nomatch=NULL]
  17
  18 df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
  19           insult_pred = insult_prob >= 0.5,
  20           profanity_pred = profanity_prob >= 0.5,
  21           severe_toxicity_pred = severe_toxicity_prob >= 0.5,
  22           threat_pred = threat_prob >= 0.5,
  23           toxicity_pred = toxicity_prob >= 0.5,
  24           identity_attack_coded = identity_attack >= 0.5,
  25           insult_coded = insult >= 0.5,
  26           profanity_coded = obscene >= 0.5,
  27           severe_toxicity_coded = severe_toxicity >= 0.5,
  28           threat_coded = threat >= 0.5,
  29           toxicity_coded = toxicity >= 0.5
  30           )]
  31
  32 gt.0.5 <- function(v) { v >= 0.5 }
  33 dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}
  34
  35 df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
  36                sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
  37                religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
  38                race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity),
  39                disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]
  40
  41 df <- df[,white:=gt.0.5(white)]