# Community Data Science Collective R Utilities
#
# Copyright (c) 2010-2016 Benjamin Mako Hill and Aaron Shaw 
# mako@atdot.cc, aaronshaw@northwestern.edu

# loads simple utility functions for use in the subsequent files

# store this for re-use across various scripts
wikiq.header <- c("title", "articleid", "revid", "timestamp", "anon",
                  "editor", "editor_id", "minor", "text_size",
                  "text_entropy", "text_md5", "reversion",
                  "additions_size", "deletions_size", "edits",
                  "articles", "users")

# helper function to load the TSV files our perl scripts are generating
load.extracted.df <- function (filename) {
  read.delim(filename, header=T, quote="", na.strings="", stringsAsFactors=TRUE)
}

# helper function to grab the classes of all columns of a dataframe
# keep this because it's being used but this can just be lapply(d, class)
get.col.classes <- function (d) {
  sapply(colnames(d), function (col) { class(d[,col]) })
}

# convert mediawiki timestamps into POSIXct
timestamp.to.POSIXct <- function (ts.string)  {
  ts.string <- gsub("T", " ", ts.string)
  ts.string <- gsub("Z", "", ts.string)
  return(as.POSIXct(ts.string, format="%Y-%m-%d %H:%M:%S", tz="UTC"))
}


read.wikiq <- function (con, header=TRUE, detect.reverts=FALSE) {
  d <- read.delim(con, stringsAsFactors=FALSE, header=header,
                  encoding="UTF-8", quote="")

  # rename date.time to timestamp and remove _
  colnames(d)[colnames(d) == "date.time"] <- "timestamp"
  colnames(d) <- sub("_", ".", colnames(d))
  
  d$timestamp <- as.POSIXct(sub("^(.*)y(.*)\xc8zy$", "\\1\\2",
                                d$timestamp), tz="UTC")

  # convert reversion to a logical
  d$reversion <- !is.na(d$reversion)

  if (detect.reverts) {
      # reorder so we cannow find the order and timestamp
      d <- d[order(d$title, d$timestamp),]
      
      # generate a list of reverted editors and a list of previous and next md5
      d$reverted <- c(d$reversion[2:length(d$reversion)],NA)
      d$md5.next <- c(d$text.md5[2:length(d$reversion)],NA)
      d$md5.prev <- c(NA,d$text.md5[1:(length(d$reversion)-1)])
      d$reverted <- d$reverted & (d$md5.next == d$md5.prev)

      # drop the extra columns and the last edit
      d <- d[!is.na(d$reverted),]
      d <- d[,!colnames(d) %in% c("md5.next", "md5.prev")]
  
      # create a reverted by variable by shifting up the editors and
      # then NAing nonreverts
      d$reverted.by <- c(d$editor[2:length(d$reversion)], NA)
      d$reverted.by[!d$reverted] <- NA
  }  
  # set ip address to the username and create a new variable
  d$ipaddress <- d$editor == ""
  d$editor[d$editor == ""] <- d$editor.id[d$editor == ""]
  
  # delete the connection
  return(d)
}

# TODO refactor this so that we clean the data BEFORE we read it into R
# ATM, this is set to only work on 14 item issues

# see the vereins wiki for "Philcomputing" and 29 lines that seem to
# have a newline in the editor name
read.bz.wikiq <- function (filename, header=TRUE, detect.reverts=FALSE) {
  con <- pipe(paste("bzcat", filename, "|awk -F'\t' '{if (NF == 14) print;}'"))
  d <- read.wikiq(con, header=header, detect.reverts=detect.reverts)
  rm(con)
  return(d)
}