]> code.communitydata.science - ml_measurement_error_public.git/blob - simulations/measerr_methods.R
add stuff to get perspective scores from civil comments
[ml_measurement_error_public.git] / simulations / measerr_methods.R
1 library(formula.tools)
2 library(matrixStats)
3 library(bbmle)
4 ## df: dataframe to model
5 ## outcome_formula: formula for y | x, z
6 ## outcome_family: family for y | x, z
7 ## proxy_formula: formula for w | x, z, y 
8 ## proxy_family: family for w | x, z, y
9 ## truth_formula: formula for x | z
10 ## truth_family: family for x | z
11
12 ### ideal formulas for example 1
13 # test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
14
15 ### ideal formulas for example 2
16 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
17
18
19 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
20 measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
21
22     nll <- function(params){
23         df.obs <- model.frame(outcome_formula, df)
24         proxy.variable <- all.vars(proxy_formula)[1]
25         proxy.model.matrix <- model.matrix(proxy_formula, df)
26         response.var <- all.vars(outcome_formula)[1]
27         y.obs <- with(df.obs,eval(parse(text=response.var)))
28         outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
29
30         param.idx <- 1
31         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
32         outcome.params <- params[param.idx:n.outcome.model.covars]
33         param.idx <- param.idx + n.outcome.model.covars
34
35         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
36             ll.y.obs <- vector(mode='numeric', length=length(y.obs))
37             ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
38             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
39         }
40
41         df.obs <- model.frame(proxy_formula,df)
42         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
43         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
44
45         param.idx <- param.idx + n.proxy.model.covars
46         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
47
48         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
49             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
50             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
51             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
52         }
53
54         ll.obs <- sum(ll.y.obs + ll.w.obs)
55
56         df.unobs <- df[is.na(df[[response.var]])]
57         df.unobs.y1 <- copy(df.unobs)
58         df.unobs.y1[[response.var]] <- 1
59         df.unobs.y0 <- copy(df.unobs)
60         df.unobs.y0[[response.var]] <- 0
61         
62         ## integrate out y
63         outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
64
65         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
66             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
67             ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
68             ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
69             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
70         }
71
72         proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
73         proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
74         proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
75
76         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
77             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
78             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
79             ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
80             ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
81
82             ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
83             ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
84         }
85
86         ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
87         ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
88         ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
89         ll <- ll.unobs + ll.obs
90         return(-ll)
91     }
92     
93     params <- colnames(model.matrix(outcome_formula,df))
94     lower <- rep(-Inf, length(params))
95     proxy.params <- colnames(model.matrix(proxy_formula, df))
96     params <- c(params, paste0('proxy_',proxy.params))
97     lower <- c(lower, rep(-Inf, length(proxy.params)))
98     start <- rep(0.1,length(params))
99     names(start) <- params
100     
101     if(method=='optim'){
102         fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
103     } else {
104         quoted.names <- gsub("[\\(\\)]",'',names(start))
105         print(quoted.names)
106         text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
107
108         measerr_mle_nll <- eval(parse(text=text))
109         names(start) <- quoted.names
110         names(lower) <- quoted.names
111         fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
112     }
113     return(fit)
114 }
115
116 ## Experimental, and not necessary if errors are independent.
117 measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
118
119     ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
120
121     ## probability of y given observed data.
122     df.obs <- df[!is.na(x.obs.1)]
123     proxy.variable <- all.vars(proxy_formula)[1]
124     df.x.obs.1 <- copy(df.obs)[,x:=1]
125     df.x.obs.0 <- copy(df.obs)[,x:=0]
126     y.obs <- df.obs[,y]
127
128     nll <- function(params){
129         outcome.model.matrix.x.obs.0 <- model.matrix(outcome_formula, df.x.obs.0)
130         outcome.model.matrix.x.obs.1 <- model.matrix(outcome_formula, df.x.obs.1)
131
132         param.idx <- 1
133         n.outcome.model.covars <- dim(outcome.model.matrix.x.obs.0)[2]
134         outcome.params <- params[param.idx:n.outcome.model.covars]
135         param.idx <- param.idx + n.outcome.model.covars
136
137         sigma.y <- params[param.idx]
138         param.idx <- param.idx + 1
139
140         ll.y.x.obs.0 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.0),sd=sigma.y, log=TRUE)
141         ll.y.x.obs.1 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.1),sd=sigma.y, log=TRUE)
142
143         ## assume that the two coders are statistically independent conditional on x
144         ll.x.obs.0.x0 <- vector(mode='numeric', length=nrow(df.obs))
145         ll.x.obs.1.x0 <- vector(mode='numeric', length=nrow(df.obs))
146         ll.x.obs.0.x1 <- vector(mode='numeric', length=nrow(df.obs))
147         ll.x.obs.1.x1 <- vector(mode='numeric', length=nrow(df.obs))
148
149         rater.model.matrix.x.obs.0 <- model.matrix(rater_formula, df.x.obs.0)
150         rater.model.matrix.x.obs.1 <- model.matrix(rater_formula, df.x.obs.1)
151
152         n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
153         rater.0.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
154         param.idx <- param.idx + n.rater.model.covars
155
156         rater.1.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
157         param.idx <- param.idx + n.rater.model.covars
158         
159         # probability of rater 0 if x is 0 or 1
160         ll.x.obs.0.x0[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
161         ll.x.obs.0.x0[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
162         ll.x.obs.0.x1[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==1,]), log=TRUE)
163         ll.x.obs.0.x1[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
164
165         # probability of rater 1 if x is 0 or 1
166         ll.x.obs.1.x0[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==1,]), log=TRUE)
167         ll.x.obs.1.x0[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
168         ll.x.obs.1.x1[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==1,]), log=TRUE)
169         ll.x.obs.1.x1[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
170
171         proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.x.obs.0)
172         proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.x.obs.1)
173
174         n.proxy.model.covars <- dim(proxy.model.matrix.x0)[2]
175         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
176         param.idx <- param.idx + n.proxy.model.covars
177
178         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
179
180         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
181             ll.w.obs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
182             ll.w.obs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
183
184                                         # proxy_formula likelihood using logistic regression
185             ll.w.obs.x0[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==1,]),log=TRUE)
186             ll.w.obs.x0[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
187
188             ll.w.obs.x1[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==1,]),log=TRUE)
189             ll.w.obs.x1[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
190         }
191
192         ## assume that the probability of x is a logistic regression depending on z
193         truth.model.matrix.obs <- model.matrix(truth_formula, df.obs)
194         n.truth.params <- dim(truth.model.matrix.obs)[2]
195         truth.params <- params[param.idx:(n.truth.params + param.idx - 1)]
196
197         ll.obs.x0 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE)
198         ll.obs.x1 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE, lower.tail=FALSE)
199
200         ll.obs <- colLogSumExps(rbind(ll.y.x.obs.0 + ll.x.obs.0.x0 + ll.x.obs.1.x0 + ll.obs.x0 + ll.w.obs.x0,
201                                       ll.y.x.obs.1 + ll.x.obs.0.x1 + ll.x.obs.1.x1 + ll.obs.x1 + ll.w.obs.x1))
202
203         ### NOW FOR THE FUN PART. Likelihood of the unobserved data.
204         ### we have to integrate out x.obs.0, x.obs.1, and x.
205
206
207         ## THE OUTCOME
208         df.unobs <- df[is.na(x.obs)]
209         df.x.unobs.0 <- copy(df.unobs)[,x:=0]
210         df.x.unobs.1 <- copy(df.unobs)[,x:=1]
211         y.unobs <- df.unobs$y
212
213         outcome.model.matrix.x.unobs.0 <- model.matrix(outcome_formula, df.x.unobs.0)
214         outcome.model.matrix.x.unobs.1 <- model.matrix(outcome_formula, df.x.unobs.1)
215
216         ll.y.unobs.x0 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.0), sd=sigma.y, log=TRUE)
217         ll.y.unobs.x1 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.1), sd=sigma.y, log=TRUE)
218
219         
220         ## THE UNLABELED DATA
221
222         
223         ## assume that the two coders are statistically independent conditional on x
224         ll.x.unobs.0.x0 <- vector(mode='numeric', length=nrow(df.unobs))
225         ll.x.unobs.1.x0 <- vector(mode='numeric', length=nrow(df.unobs))
226         ll.x.unobs.0.x1 <- vector(mode='numeric', length=nrow(df.unobs))
227         ll.x.unobs.1.x1 <- vector(mode='numeric', length=nrow(df.unobs))
228         
229         df.x.unobs.0[,x.obs := 1]
230         df.x.unobs.1[,x.obs := 1]
231
232         rater.model.matrix.x.unobs.0 <- model.matrix(rater_formula, df.x.unobs.0)
233         rater.model.matrix.x.unobs.1 <- model.matrix(rater_formula, df.x.unobs.1)
234
235          
236         ## # probability of rater 0 if x is 0 or 1
237         ## ll.x.unobs.0.x0 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
238         ##                                      plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
239
240         ## ll.x.unobs.0.x1 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
241         ##                                        plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
242
243         ## # probability of rater 1 if x is 0 or 1
244         ## ll.x.unobs.1.x0 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
245         ##                                      plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
246
247         ## ll.x.unobs.1.x1 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
248         ##                                      plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
249
250
251         proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
252         proxy.model.matrix.x0.unobs <- model.matrix(proxy_formula, df.x.unobs.0)
253         proxy.model.matrix.x1.unobs <- model.matrix(proxy_formula, df.x.unobs.1)
254
255         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
256             ll.w.unobs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
257             ll.w.unobs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
258
259
260                                         # proxy_formula likelihood using logistic regression
261             ll.w.unobs.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==1,]),log=TRUE)
262             ll.w.unobs.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
263
264             ll.w.unobs.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==1,]),log=TRUE)
265             ll.w.unobs.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
266         }
267
268         truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs)
269
270         ll.unobs.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
271         ll.unobs.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
272
273         ll.unobs <- colLogSumExps(rbind(ll.unobs.x0 + ll.w.unobs.x0 + ll.y.unobs.x0,
274                                         ll.unobs.x1 + ll.w.unobs.x1 + ll.y.unobs.x1))
275
276         return(-1 *( sum(ll.obs) + sum(ll.unobs)))
277     }
278
279     outcome.params <- colnames(model.matrix(outcome_formula,df))
280     lower <- rep(-Inf, length(outcome.params))
281
282     if(outcome_family$family=='gaussian'){
283         params <- c(outcome.params, 'sigma_y')
284         lower <- c(lower, 0.00001)
285     } else {
286         params <- outcome.params
287     }
288     
289     rater.0.params <- colnames(model.matrix(rater_formula,df))
290     params <- c(params, paste0('rater_0',rater.0.params))
291     lower <- c(lower, rep(-Inf, length(rater.0.params)))
292
293     rater.1.params <- colnames(model.matrix(rater_formula,df))
294     params <- c(params, paste0('rater_1',rater.1.params))
295     lower <- c(lower, rep(-Inf, length(rater.1.params)))
296
297     proxy.params <- colnames(model.matrix(proxy_formula, df))
298     params <- c(params, paste0('proxy_',proxy.params))
299     lower <- c(lower, rep(-Inf, length(proxy.params)))
300
301     truth.params <- colnames(model.matrix(truth_formula, df))
302     params <- c(params, paste0('truth_', truth.params))
303     lower <- c(lower, rep(-Inf, length(truth.params)))
304     start <- rep(0.1,length(params))
305     names(start) <- params
306     
307     
308     if(method=='optim'){
309         fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
310     } else {
311                 
312         quoted.names <- gsub("[\\(\\)]",'',names(start))
313         print(quoted.names)
314         text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
315
316         measerr_mle_nll <- eval(parse(text=text))
317         names(start) <- quoted.names
318         names(lower) <- quoted.names
319         fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
320     }
321
322     return(fit)
323 }
324
325
326 measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
327
328     measerr_mle_nll <- function(params){
329         df.obs <- model.frame(outcome_formula, df)
330         proxy.variable <- all.vars(proxy_formula)[1]
331         proxy.model.matrix <- model.matrix(proxy_formula, df)
332         response.var <- all.vars(outcome_formula)[1]
333         y.obs <- with(df.obs,eval(parse(text=response.var)))
334
335         outcome.model.matrix <- model.matrix(outcome_formula, df)
336
337         param.idx <- 1
338         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
339         outcome.params <- params[param.idx:n.outcome.model.covars]
340         param.idx <- param.idx + n.outcome.model.covars
341
342         ## likelihood for the fully observed data 
343         if(outcome_family$family == "gaussian"){
344             sigma.y <- params[param.idx]
345             param.idx <- param.idx + 1
346
347                                         #  outcome_formula likelihood using linear regression
348             ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
349         }
350         
351         df.obs <- model.frame(proxy_formula,df)
352         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
353         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
354         param.idx <- param.idx + n.proxy.model.covars
355         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
356
357         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
358             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
359
360                                         # proxy_formula likelihood using logistic regression
361             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
362             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
363         }
364
365         df.obs <- model.frame(truth_formula, df)
366         truth.variable <- all.vars(truth_formula)[1]
367         truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
368         truth.model.matrix <- model.matrix(truth_formula,df)
369         n.truth.model.covars <- dim(truth.model.matrix)[2]
370         
371         truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
372
373         if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
374             ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
375
376                                         # truth_formula likelihood using logistic regression
377             ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
378             ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
379         }
380         
381                                         # add the three likelihoods
382         ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
383
384         ## likelihood for the predicted data
385         ## integrate out the "truth" variable. 
386         
387         if(truth_family$family=='binomial'){
388             df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
389             df.unobs.x1 <- copy(df.unobs)
390             df.unobs.x1[,'x'] <- 1
391             df.unobs.x0 <- copy(df.unobs)
392             df.unobs.x0[,'x'] <- 0
393             outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
394             
395             outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
396             outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
397             if(outcome_family$family=="gaussian"){
398
399                                         # likelihood of outcome
400                 ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
401                 ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
402             }
403
404             if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
405
406                 proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
407                 proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
408                 proxy.unobs <- df.unobs[[proxy.variable]]
409                 ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
410                 ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
411
412                                         # likelihood of proxy
413                 ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
414                 ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
415
416                 ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
417                 ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
418             }
419
420             if(truth_family$link=='logit'){
421                 truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
422                                         # likelihood of truth
423                 ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
424                 ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
425             }
426         }
427
428         ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
429         ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
430         ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
431         return(-(ll.unobs + ll.obs))
432     }
433     
434     outcome.params <- colnames(model.matrix(outcome_formula,df))
435     lower <- rep(-Inf, length(outcome.params))
436
437     if(outcome_family$family=='gaussian'){
438         params <- c(outcome.params, 'sigma_y')
439         lower <- c(lower, 0.00001)
440     } else {
441         params <- outcome.params
442     }
443     
444     proxy.params <- colnames(model.matrix(proxy_formula, df))
445     params <- c(params, paste0('proxy_',proxy.params))
446     lower <- c(lower, rep(-Inf, length(proxy.params)))
447
448     truth.params <- colnames(model.matrix(truth_formula, df))
449     params <- c(params, paste0('truth_', truth.params))
450     lower <- c(lower, rep(-Inf, length(truth.params)))
451     start <- rep(0.1,length(params))
452     names(start) <- params
453
454     if(method=='optim'){
455         fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
456     } else { # method='mle2'
457                 
458         quoted.names <- gsub("[\\(\\)]",'',names(start))
459
460         text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
461
462         measerr_mle_nll_mle <- eval(parse(text=text))
463         names(start) <- quoted.names
464         names(lower) <- quoted.names
465         fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
466     }
467
468     return(fit)
469 }
470

Community Data Science Collective || Want to submit a patch?