]> code.communitydata.science - ml_measurement_error_public.git/blob - simulations/measerr_methods.R
Add simulation of listwise deletion and averaging of labeled-only estimators
[ml_measurement_error_public.git] / simulations / measerr_methods.R
1 library(formula.tools)
2 library(matrixStats)
3
4 ## df: dataframe to model
5 ## outcome_formula: formula for y | x, z
6 ## outcome_family: family for y | x, z
7 ## proxy_formula: formula for w | x, z, y 
8 ## proxy_family: family for w | x, z, y
9 ## truth_formula: formula for x | z
10 ## truth_family: family for x | z
11
12 ### ideal formulas for example 1
13 # test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
14
15 ### ideal formulas for example 2
16 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
17
18
19 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
20 measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
21
22     nll <- function(params){
23         df.obs <- model.frame(outcome_formula, df)
24         proxy.variable <- all.vars(proxy_formula)[1]
25         proxy.model.matrix <- model.matrix(proxy_formula, df)
26         response.var <- all.vars(outcome_formula)[1]
27         y.obs <- with(df.obs,eval(parse(text=response.var)))
28         outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
29
30         param.idx <- 1
31         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
32         outcome.params <- params[param.idx:n.outcome.model.covars]
33         param.idx <- param.idx + n.outcome.model.covars
34
35         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
36             ll.y.obs <- vector(mode='numeric', length=length(y.obs))
37             ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
38             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
39         }
40
41         df.obs <- model.frame(proxy_formula,df)
42         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
43         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
44
45         param.idx <- param.idx + n.proxy.model.covars
46         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
47
48         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
49             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
50             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
51             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
52         }
53
54         ll.obs <- sum(ll.y.obs + ll.w.obs)
55
56         df.unobs <- df[is.na(df[[response.var]])]
57         df.unobs.y1 <- copy(df.unobs)
58         df.unobs.y1[[response.var]] <- 1
59         df.unobs.y0 <- copy(df.unobs)
60         df.unobs.y0[[response.var]] <- 1
61         
62         ## integrate out y
63         outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
64
65         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
66             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
67             ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
68             ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
69             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
70         }
71
72         proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
73         proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
74         proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
75
76         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
77             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
78             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
79             ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
80             ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
81
82             ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
83             ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
84         }
85
86         ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
87         ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
88         ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
89         ll <- ll.unobs + ll.obs
90         return(-ll)
91     }
92     
93     params <- colnames(model.matrix(outcome_formula,df))
94     lower <- rep(-Inf, length(params))
95     proxy.params <- colnames(model.matrix(proxy_formula, df))
96     params <- c(params, paste0('proxy_',proxy.params))
97     lower <- c(lower, rep(-Inf, length(proxy.params)))
98     start <- rep(0.1,length(params))
99     names(start) <- params
100     
101     fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
102     return(fit)
103 }
104
105 measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
106
107     measrr_mle_nll <- function(params){
108         df.obs <- model.frame(outcome_formula, df)
109         
110         proxy.variable <- all.vars(proxy_formula)[1]
111         proxy.model.matrix <- model.matrix(proxy_formula, df)
112
113         response.var <- all.vars(outcome_formula)[1]
114         y.obs <- with(df.obs,eval(parse(text=response.var)))
115         
116         outcome.model.matrix <- model.matrix(outcome_formula, df)
117
118         param.idx <- 1
119         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
120         outcome.params <- params[param.idx:n.outcome.model.covars]
121         param.idx <- param.idx + n.outcome.model.covars
122
123         ## likelihood for the fully observed data 
124         if(outcome_family$family == "gaussian"){
125             sigma.y <- params[param.idx]
126             param.idx <- param.idx + 1
127             ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
128         }
129         
130         df.obs <- model.frame(proxy_formula,df)
131         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
132         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
133         param.idx <- param.idx + n.proxy.model.covars
134         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
135
136         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
137             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
138             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
139             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
140         }
141
142         df.obs <- model.frame(truth_formula, df)
143         truth.variable <- all.vars(truth_formula)[1]
144         truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
145         truth.model.matrix <- model.matrix(truth_formula,df)
146         n.truth.model.covars <- dim(truth.model.matrix)[2]
147         
148         truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
149
150         if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
151             ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
152             ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
153             ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
154         }
155         
156         ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
157
158         ## likelihood for the predicted data
159         ## integrate out the "truth" variable. 
160         
161         if(truth_family$family=='binomial'){
162             df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
163             df.unobs.x1 <- copy(df.unobs)
164             df.unobs.x1[,'x'] <- 1
165             df.unobs.x0 <- copy(df.unobs)
166             df.unobs.x0[,'x'] <- 0
167             outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
168             
169             outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
170             outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
171             if(outcome_family$family=="gaussian"){
172             ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
173             ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
174             }
175
176             if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
177
178                 proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
179                 proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
180                 proxy.unobs <- df.unobs[[proxy.variable]]
181                 ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
182                 ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
183
184                 ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
185                 ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
186
187                 ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
188                 ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
189             }
190
191             if(truth_family$link=='logit'){
192                 truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
193                 ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
194                 ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
195             }
196         }
197
198         ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
199         ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
200         ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
201         return(-(ll.unobs + ll.obs))
202     }
203     
204     outcome.params <- colnames(model.matrix(outcome_formula,df))
205     lower <- rep(-Inf, length(outcome.params))
206
207     if(outcome_family$family=='gaussian'){
208         params <- c(outcome.params, 'sigma_y')
209         lower <- c(lower, 0.00001)
210     } else {
211         params <- outcome.params
212     }
213     
214     proxy.params <- colnames(model.matrix(proxy_formula, df))
215     params <- c(params, paste0('proxy_',proxy.params))
216     lower <- c(lower, rep(-Inf, length(proxy.params)))
217
218     truth.params <- colnames(model.matrix(truth_formula, df))
219     params <- c(params, paste0('truth_', truth.params))
220     lower <- c(lower, rep(-Inf, length(truth.params)))
221     start <- rep(0.1,length(params))
222     names(start) <- params
223     
224     fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
225
226     return(fit)
227 }

Community Data Science Collective || Want to submit a patch?