library(data.table)
library(MASS)

set.seed(1111)

scores <- fread("perspective_scores.csv")
scores <- scores[,id:=as.character(id)]

df <- fread("all_data.csv")

# only use the data that has identity annotations
df <- df[identity_annotator_count > 0]

(df[!(df$id %in% scores$id)])

df <- df[scores,on='id',nomatch=NULL]

df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
          insult_pred = insult_prob >= 0.5,
          profanity_pred = profanity_prob >= 0.5,
          severe_toxicity_pred = severe_toxicity_prob >= 0.5,
          threat_pred = threat_prob >= 0.5,
          toxicity_pred = toxicity_prob >= 0.5,
          identity_attack_coded = identity_attack >= 0.5,
          insult_coded = insult >= 0.5,
          profanity_coded = obscene >= 0.5,
          severe_toxicity_coded = severe_toxicity >= 0.5,
          threat_coded = threat >= 0.5,
          toxicity_coded = toxicity >= 0.5
          )]

gt.0.5 <- function(v) { v >= 0.5 }
dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}

df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
               sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
               religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
               race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity), 
               disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]

df <- df[,white:=gt.0.5(white)]