1 # Functions for creating samples of datasets
2 # Copyright (C) 2018 Nathan TeBlunthuis
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
17 sample.by.wiki <- function(dt,outcome,N.per.wiki=30){
19 sample.params <- dt[,.N,by=wiki.name]
20 sample.params[,group.sd := dt[,.(group.sd=sd(.SD[[outcome]])),by=wiki.name]$group.sd]
22 sample.params[,p.in.group := N / nrow(dt)]
24 sample.params[,min.N := min(N)]
25 sample.params[,n.from.group := pmin(min.N/(1-group.sd), N)]
27 sample.params[,p.sampled := n.from.group/N]
29 sample.params[,weight := 1/p.sampled]
30 dt[sample.params,":="(prob=p.sampled,weight=weight),on=.(wiki.name)]
31 sample.idx <- sample(nrow(dt),size=sum(sample.params$n.from.group,na.rm=TRUE),prob=dt$prob)
33 return(dt[sample.idx])
36 sample.newcomers <- function()
38 wikis.to.remove <- newcomers[,.N,by="wiki.name"][N<30]$wiki.name
39 remember(nrow(wikis.to.remove),"n.wikis.insufficient.newcomers")
40 newcomers.presample <- newcomers[!(wiki.name %in% wikis.to.remove)]
41 newcomers.sample <- sample.by.wiki(newcomers.presample,"survives")
42 return(newcomers.sample)
45 sample.ns4.edits <- function(){
46 wikis.to.keep <- ns4.reg.edits[,.(.N,N.reverts=sum(reverted)),by=wiki.name][(N>30)&(N.reverts > 30)]
47 ns4.reg.edits.sub <- ns4.reg.edits[wiki.name %in% wikis.to.keep$wiki.name]
48 ns4.reg.edits.sample <- sample.by.wiki(ns4.reg.edits.sub,"reverted")
49 return(ns4.reg.edits.sample)
52 sample.wiki.data <- function(){
53 ## just choose 100 random wikis
54 wikis.to.keep <- sample(unique(wiki.data$wiki.name),100)
55 wiki.data.sample <- wiki.data[wiki.name %in% wikis.to.keep]
56 return(wiki.data.sample)