code/data_processing/05_save_descriptives.R

   1 df = read.csv('processed_data/abstracts.tsv',sep='\t', strip.white=TRUE)
   2 df['date'] = as.Date(df$date)
   3 df$modal_country[df['modal_country'] == ''] <- NA
   4 df['year'] = format(df['date'],'%Y')
   5
   6 abstracts <- df[df['abstract'] != '',c('eid','abstract')]
   7 # Creates a vector of word counts, based on counting all of the groups of alphanumeric characters
   8 word_count <- apply(abstracts, 1, function(x) sapply(gregexpr("[[:alnum:]]+", x['abstract']), function(x) sum(x > 0)))
   9
  10 s = read.csv('processed_data/paper_subject_table.tsv', sep='\t')
  11 full <- merge(df,s, by.x = 'eid', by.y = 'paper_eid')
  12
  13 # zero these out before we save them so we don't save all of the abstracts.
  14 full['abstract'] <- NULL
  15 df['abstract'] <- NULL
  16
  17 save(df, abstracts, s, full, word_count, file="paper/data/orig_data_sets.RData")